llp: add s3 lambda to tag files for expiry

This change adds in a lambda function and the
terraform to upload and enable it that will check
if an uploaded (created) file is in a protected
path.  If not, the S3 object will be tagged for
eventual deletion via a lifecycle rule when it's
lifespan expires.

Signed-off-by: Kelley Spoon <kelley.spoon@linaro.org>
Change-Id: I34ade1f38311a269c479d6373495a789de96ab15
Reviewed-on: https://review.linaro.org/c/infrastructure/linaro-license-protection/+/40186

7 files changed, 223 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index 6a6ebd0..6908226 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,4 @@ download_report*.csv
 secrets.py
 allowed_hosts.txt
 local_settings.py
+*lambda.zip
diff --git a/s3/build.sh b/s3/build.sh
new file mode 100755
index 0000000..54ad446
--- /dev/null
+++ b/s3/build.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+TARG=llp-tag-expires
+
+rm -f $TARG/lambda.zip
+(cd $TARG && pipenv run lambpy -r requirements.txt)
diff --git a/s3/llp-tag-expires/Pipfile b/s3/llp-tag-expires/Pipfile
new file mode 100644
index 0000000..887d86c
--- /dev/null
+++ b/s3/llp-tag-expires/Pipfile
@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+boto3 = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.8"
diff --git a/s3/llp-tag-expires/Pipfile.lock b/s3/llp-tag-expires/Pipfile.lock
new file mode 100644
index 0000000..02cefe1
--- /dev/null
+++ b/s3/llp-tag-expires/Pipfile.lock
@@ -0,0 +1,77 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "0ba145c19353da73840755ed85984b6653241c800c6ad2c772805a6089dfb424"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.8"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "boto3": {
+            "hashes": [
+                "sha256:76b3ee0d1dd860c9218bc864cd29f1ee986f6e1e75e8669725dd3c411039379e",
+                "sha256:c39cb6ed376ba1d4689ac8f6759a2b2d8a0b0424dbec0cd3af1558079bcf06e8"
+            ],
+            "index": "pypi",
+            "version": "==1.20.23"
+        },
+        "botocore": {
+            "hashes": [
+                "sha256:640b62110aa6d1c25553eceafb5bcd89aedeb84b191598d1f6492ad24374d285",
+                "sha256:7459766c4594f3b8877e8013f93f0dc6c6486acbeb7d9c9ae488396529cc2e84"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.23.23"
+        },
+        "jmespath": {
+            "hashes": [
+                "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9",
+                "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==0.10.0"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==2.8.2"
+        },
+        "s3transfer": {
+            "hashes": [
+                "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c",
+                "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.5.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==1.16.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece",
+                "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.7"
+        }
+    },
+    "develop": {}
+}
diff --git a/s3/llp-tag-expires/llp-tag-expires.py b/s3/llp-tag-expires/llp-tag-expires.py
new file mode 100644
index 0000000..1a71599
--- /dev/null
+++ b/s3/llp-tag-expires/llp-tag-expires.py
@@ -0,0 +1,56 @@
+#!/usr/bin/python3
+
+import json
+import urllib.parse
+import boto3
+from datetime import datetime, timedelta
+import fnmatch
+
+EXPIRE_TAG = "LLP_Expires"
+EXPIRE_TIME = timedelta(days=180)
+PROTECTED_PATHS = [
+    'releases/*',
+    'snapshots/gnu-toolchain/*',
+    'snapshots/android/binaries/*',
+    'snapshots/components/kernel/leg-96boards-developerbox-edk2/*',
+    'snapshots/components/toolchain/infrastructure/*',
+    'snapshots/components/toolchain/gcc-linaro/*',
+    'snapshots/components/toolchain/binaries/*',
+    'snapshots/96boards/*/binaries/*',
+    'snapshots/96boards/*/linaro/debian/*',
+    'snapshots/components/pyarmnn-tests/*',
+    'snapshots/android/lkft/protected/*BUILD-INFO',
+]
+
+s3 = boto.client('s3')
+now = datetime.utcnow()
+
+def lambda_handler(event, context):
+    print("Received event" + json.dumps(event, indent=2))
+
+    for record in event['Records']:
+        bucket = record['s3']['bucket']['name']
+        key = urllib.parse.unquote_plus(record['s3']['object']['key'],
+            encoding='utf-8')
+
+        # check if key in protected fields and exit if yes
+        for protected in PROTECTED_PATHS:
+            if fnmatch(key, protected):
+                print(f"Key {key} protected by rule {protected}")
+                return
+
+        # No rule to protect the filepath, so let's tag it for expiration
+        try:
+            response = s3.put_object_tagging(
+                Bucket = bucket,
+                Tagging = {
+                    'TagSet': [
+                        'Key': EXPIRE_TAG,
+                        'Value': str(now.date()+EXPIRE_TIME)
+                    ]
+                }
+            )
+        except Exception as e:
+            print(f"Error setting tag for {key}")
+            print(e)
+        raise e
diff --git a/s3/llp-tag-expires/requirements.txt b/s3/llp-tag-expires/requirements.txt
new file mode 100644
index 0000000..30ddf82
--- /dev/null
+++ b/s3/llp-tag-expires/requirements.txt
@@ -0,0 +1 @@
+boto3
diff --git a/s3/tag-expires.tf b/s3/tag-expires.tf
new file mode 100644
index 0000000..1c7778d
--- /dev/null
+++ b/s3/tag-expires.tf
@@ -0,0 +1,70 @@
+# S3 bucket was setup manually, and we don't want to manage it here
+data "aws_s3_bucket" "bucket" {
+}
+
+resource "aws_lambda_function" "s3lambda" {
+  filename      = "${path.module}/llp-tag-expires/lambda.zip"
+  function_name = "llp-tag-expires"
+  role          = aws_iam_role.s3_lambda_role.arn
+  handler       = "index.lambda_handler"
+  runtime       = "python3.8"
+  depends_on    = [aws_iam_role_policy_attachment.logging_policy_attach]
+}
+
+
+resource "aws_s3_bucket_notification" "s3_bucket_notification" {
+    bucket = data.bucket.id
+    lambda_function {
+        lambda_function_arn = aws_lambda_function.s3lambda.arn
+        events              = ["s3:ObjectCreated:*"]
+    }
+    depends_on = [
+        aws_iam_role_policy_attachment.logging_policy_attach,
+        aws_lambda_permission.allow_bucket,
+        aws_lambda_function.s3lambda
+    ]
+}
+
+resource "aws_iam_role" "s3_lambda_role" {
+    name               = "s3_lambda_function_role"
+    assume_role_policy = <<EOF
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+            "Action": "sts:AssumeRole",
+            "Principal": {
+                "Service": "lambda.amazonaws.com"
+            },
+            "Effect": "Allow"
+            }
+        ]
+    }
+    EOF
+}
+
+
+# IAM policy for logging from a lambda
+resource "aws_iam_policy" "lambda_tagging" {
+    name        = "LambdaTaggingPolicy"
+    path        = "/"
+    description = "Tags files to be expired based on paths"
+    policy      = <<EOF
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "VisualEditor0",
+                "Effect": "Allow",
+                "Action": "s3:PutObjectTagging",
+                "Resource": "${data.bucket.arn}/*"
+            },
+        ]
+    }
+    EOF
+}
+
+resource "aws_iam_role_policy_attachment" "tagging_policy_attach" {
+    role       = aws_iam_role.s3_lambda_role.name
+    policy_arn = aws_iam_policy.lambda_tagging.arn
+}