# Author: Paritosh Walvekar
# Email: paritosw@amazon.com
# Date: November 19, 2020

Transform: AWS::Serverless-2016-10-31
Description: "Template to create databrew architecture."

Parameters:

  # Name of the S3 bucket which will store the dataset and the partitioned files and processed files.
  S3Bucket:
    Description: "The S3 bucket to store the code for the lambda functions and archives for lambda layers"
    Type: String
    AllowedPattern : ".+"

Resources:

  # S3 bucket to store the dataset.
  CSVBucket:
    Type: AWS::S3::Bucket
  
  #SplitPandas lambda function.
  SplitPandas:
    Type: AWS::Serverless::Function
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - "sts:AssumeRole"
      Policies:
        - Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - s3:GetObject
                - s3:PutObject
              Resource: "*"
        - Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - sqs:SendMessage
              Resource: !GetAtt SplitPandasDumpSQSQueue.Arn
      Events:
        S3ObjectPutEvent: 
          Type: S3
          Properties:
            Bucket: !Ref CSVBucket
            Events: "s3:ObjectCreated:*"
            Filter:
              S3Key:
                Rules:
                  - Name: prefix
                    Value: "input/"
      Runtime: "python3.8"
      Handler: "lambda_function.lambda_handler"
      CodeUri:
        Bucket: !Ref S3Bucket
        Key: "splitpandas-function-code.zip"
      Environment:
        Variables:
          QUEUE_URL: !Ref SplitPandasDumpSQSQueue
      Layers:
        - !Ref SplitPandasLayer
      Timeout: 900
      MemorySize: 3008
  
  # SplitPandas lambda function layer.
  SplitPandasLayer:
    Type: AWS::Serverless::LayerVersion
    Properties:
      Description: "The lambda layer for the split pandas function."
      ContentUri:
        Bucket: !Ref S3Bucket
        Key: "splitpandas-layer-archive.zip"
  
  # SQS queue for the split pandas function to dump partitioned file S3 keys.
  SplitPandasDumpSQSQueue:
    Type: AWS::SQS::Queue
    Properties:
      VisibilityTimeout: 900     
  
  # Vader Lambda function.
  VaderFunction:
    Type: AWS::Serverless::Function
    Properties:
      AssumeRolePolicyDocument:
          Version: "2012-10-17"
          Statement:
            - Effect: Allow
              Principal:
                Service:
                  - lambda.amazonaws.com
              Action:
                - "sts:AssumeRole"
      Policies:
        - Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - s3:PutObject
                - s3:GetObject
              Resource: "*"
      Events:
        SQSMessageEvent:
          Type: SQS
          Properties:
            BatchSize: 1
            Enabled: True
            Queue: !GetAtt SplitPandasDumpSQSQueue.Arn
      Runtime: "python3.8"
      Handler: "lambda_function.lambda_handler"
      CodeUri:
        Bucket: !Ref S3Bucket
        Key: "vaderfunction-v2-function-code.zip"
      Layers:
        - !Ref VaderFunctionLayer
      Timeout: 900
      MemorySize: 3008
  
  # Vader function lambda layer.
  VaderFunctionLayer:
    Type: AWS::Serverless::LayerVersion
    Properties:
      Description: "The lambda layer for the vader function."
      ContentUri:
        Bucket: !Ref S3Bucket
        Key: "vaderfunction-v2-layer-archive.zip"
  
  # IAM role that grants DataBrew access to Glue.
  DataBrewDataAccessRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - databrew.amazonaws.com
            Action:
              - "sts:AssumeRole"
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/AmazonS3FullAccess"
        - "arn:aws:iam::aws:policy/AmazonAthenaFullAccess"
        - "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
      Policies:
        - PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - glue:GetDatabases
                  - glue:GetPartitions
                  - glue:GetTable
                  - glue:GetTables
                  - glue:GetConnection
                Resource: "*"
          PolicyName: "AWSGlueDataBrewDataAccessPolicy"
      RoleName: !Sub
        - 'DataBrewAccessRole-Stack-${StackName}'
        - { StackName: !Ref AWS::StackName }

        
# Outputs of the Cloudformation template.
Outputs:
  S3BucketName:
    Description: "Name of the S3 bucket to store CSV dataset"
    Value: !Ref CSVBucket
  SplitPandasLambdaFunctionName:
    Description: "Name of the lambda function that splits the CSV file."
    Value: !Ref SplitPandas
  SQSQueueForParitionedFileNames:
    Description: "Queue that spliting function dumps partitioned file names into."
    Value: !GetAtt SplitPandasDumpSQSQueue.QueueName
  VaderFunctionName:
    Description: "Name of the vader lambda function."
    Value: !Ref VaderFunction
  DataBrewGlueDataAccessRole:
    Description: "The IAM role that grants DataBrew access to Glue."
    Value: !Ref DataBrewDataAccessRole