import sys
import json
from datetime import datetime, timezone, timedelta
import boto3

from awsglue.utils import getResolvedOptions


def initiate_logger(now_ts, job_name, job_env):
    logger = []
    logger.append(f"Running job: {job_name}")
    logger.append(f"Timestamp job start: {now_ts}")
    logger.append(f"Environment: {job_env}")
    logger.append("************")
    return logger


def check_log_path_add_slash(arg_log_path):
    # Function checks the path (string) provided as input param and adds a '/' if missing, as per AWS best practices
    # paths should end with a slash
    if arg_log_path[-1] != '/':
        arg_log_path += '/'

    return arg_log_path


# Expected format of metadata json file:
# {
#     "version": "1.0",
#     "deleter": [
#         {
#             "environment": "PRD",
#             "project": "My-Project",
#             "bucket": "my-projects-bucket",
#             "path_to_delete": "path/to/delete/",
#             "retain_days": 1750
#         },
#         {
#             ...
#         }
#     ]
# }
def read_metadata_file(s3_client, json_bucket, json_path):
    json_metadata = s3_client.get_object(Bucket=json_bucket, Key=json_path)
    json_contents = json.loads(json_metadata['Body'].read().decode('utf-8'))
    return json_contents

    # with open(path_to_json, 'r', encoding="utf8") as file:
    # json_file = json.load(file)
    # return json_file


def check_listing_response_fill_list(page, response_container_type, retention_start_date):
    # Check and compare each object in the provided s3 objects page (param: page) if its 'LastModified' datetime
    # parameter is older than the specified retention start date. If yes - object key is added to a list.
    objects_list = []
    for obj in page[response_container_type]:
        key = obj['Key']
        ver = obj['VersionId']
        last_modified = obj['LastModified']
        if last_modified < retention_start_date:
            objects_list.append((key, ver))

    return objects_list


def delete_objects(s3_client, bucket_name, objects_list):
    # Divide the list to chunks of 1000 objects each, as delete_objects function accepts max 1000 per call
    # Use Python list comprehension
    chunks = [objects_list[x:x + 1000] for x in range(0, len(objects_list), 1000)]

    # Create a list of dictionary elements for each file in the sublist(chunk)
    for chunk in chunks:
        objects = []
        for keyId, verId in chunk:
            objects.append(
                {
                    'Key': keyId,
                    'VersionId': verId
                }
            )
        # Actual deletion of objects. Supports two modes for the response: verbose and quiet In quiet mode the response
        # includes only keys where the deletion failed. If success - no info returned.
        # Default mode is Verbose (info is returned for each object)
        s3_client.delete_objects(
            Bucket=bucket_name,
            Delete={
                'Objects': objects,
                'Quiet': True
            }
        )


def save_log_file(s3_resource, log_bucket, log_path, job_name, log_file_contents):
    save_ts = datetime.timestamp(datetime.now(timezone.utc))
    log_file_name = f"{log_path}{job_name}/{job_name}.{save_ts}"
    log_obj = s3_resource.Object(log_bucket, log_file_name)
    log_file_body = "\n".join(log_file_contents)
    log_obj.put(Body=log_file_body)
    return


def run():
    print("Starting job execution...")
    now_ts = datetime.now(timezone.utc)

    # Load Job Parameters
    args = getResolvedOptions(sys.argv, [
        'ENV',
        'JOB',
        'JSON_BUCKET',
        'JSON_FILE_PATH',
        'LOG_BUCKET',
        'LOG_PATH'
    ])

    print("Parsed Job args: ")
    for k, v in args.items():
        print(f"{k}: {v}")

    job_env = args['ENV']
    job_name = args['JOB']
    json_metadata_bucket = args['JSON_BUCKET']
    json_metadata_file_path = args['JSON_FILE_PATH']
    log_bucket = args['LOG_BUCKET']
    log_path = check_log_path_add_slash(args['LOG_PATH'])

    s3_resource = boto3.resource('s3')
    s3_client = boto3.client('s3', region_name='eu-west-1')
    paginator = s3_client.get_paginator('list_object_versions')

    error_count = 0
    log_data = initiate_logger(now_ts, job_name, job_env)
    deleter_entities = read_metadata_file(s3_client, json_metadata_bucket, json_metadata_file_path)

    # Max Retention period is 5 years.
    # For each entity in metadata file, the respective location (and its subfolders, if any) are iterated and files with
    # LastModified date, older than specified retention_start_date are collected in a list, together with their
    # VersionIds. Then list is passed to the deletion function which deletes the file and its particular version, thus
    # file is permanently deleted, no previous version and delete marker are created (for s3 buckets with versioning)
    # Also a check for any leftover Delete markers older than retention_start_date is done - if any are found, they
    # are also to be deleted.
    for entity in deleter_entities["deleter"]:
        if job_env.upper() == ((entity["environment"]).upper()):
            try:
                project = entity["project"]
                bucket = entity["bucket"]
                prefix = entity["path_to_delete"]
                retain_days = entity["retain_days"]

                # Note: Timedelta calculation is end-date exclusive
                retention_start_date = now_ts - timedelta(days=retain_days)

                print(f"Bucket: {bucket} \nPrefix: {prefix} \nRetention Start Date: {retention_start_date}")
                log_data.append(f"Bucket: {bucket}")
                log_data.append(f"Prefix: {prefix}")
                log_data.append(f"Retention Start Date: {retention_start_date}. Removing all earlier files.")

                page_iterator = paginator.paginate(Bucket=bucket, Prefix=prefix)
                deleted_objects_list = []
                delete_markers_list = []
                for page in page_iterator:
                    # Collecting list of files/objects to be deleted
                    if 'Versions' in page:
                        deleted_objects_list.extend(check_listing_response_fill_list(page, 'Versions', retention_start_date))

                    # Collecting the VersionIds of Delete markers, so that they get wiped out too.
                    if 'DeleteMarkers' in page:
                        delete_markers_list.extend(check_listing_response_fill_list(page, 'DeleteMarkers', retention_start_date))

                deleted_objects_counter = len(deleted_objects_list)
                delete_markers_counter = len(delete_markers_list)
                for obj_list in (deleted_objects_list, delete_markers_list):
                    delete_objects(s3_client, bucket_name=bucket, objects_list=obj_list)

                print(f"Deleted objects count: {deleted_objects_counter}")
                print(f"Delete markers count: {delete_markers_counter}\n")
                log_data.append(f"{deleted_objects_counter} objects successfully deleted from location.")
                log_data.append(f"{delete_markers_counter} delete markers successfully deleted from location.")
                log_data.append(f"******")
            except Exception as ex:
                print(f"EXCEPTION RAISED: {ex}")
                log_data.append(f"PROCESSING FOR ENTITY FAILED!")
                log_data.append(f"ENV: {job_env} Bucket: {entity['bucket']} Path: {entity['path_to_delete']}")
                log_data.append(f"EXCEPTION RAISED: {ex}")
                error_count += 1
        else:
            not_matched_message = f"Environment not matched for entry: ENV: {entity['environment']}, " \
                                  f"Project: {entity['project']}, bucket: {entity['bucket']}, path: {entity['path_to_delete']}\n******\n"
            print(not_matched_message)
            log_data.append(not_matched_message)

    # Log save location could be added to deleter entities metadata file, so that logs which are older than max
    # retention period (e.g. ~5 years (1750 days)) are also deleted
    save_log_file(s3_resource, log_bucket, log_path, job_name, log_data)

    # Errors processing: fail job on purpose if error_count > 0
    print(f"Number of failed operations: {str(error_count)}")
    if error_count > 0:
        print("Errors found! Terminating job!")
        sys.exit()
    else:
        print("END!")


# Program entry point
if __name__ == '__main__':
    run()

import sys
import json
from datetime import datetime, timezone, timedelta
import boto3

from awsglue.utils import getResolvedOptions


def initiate_logger(now_ts, job_name, job_env):
    logger = []
    logger.append(f"Running job: {job_name}")
    logger.append(f"Timestamp job start: {now_ts}")
    logger.append(f"Environment: {job_env}")
    logger.append("************")
    return logger


def check_log_path_add_slash(arg_log_path):
    # Function checks the path (string) provided as input param and adds a '/' if missing, as per AWS best practices
    # paths should end with a slash
    if arg_log_path[-1] != '/':
        arg_log_path += '/'

    return arg_log_path


# Expected format of metadata json file:
# {
#     "version": "1.0",
#     "deleter": [
#         {
#             "environment": "PRD",
#             "project": "My-Project",
#             "bucket": "my-projects-bucket",
#             "path_to_delete": "path/to/delete/",
#             "retain_days": 1750
#         },
#         {
#             ...
#         }
#     ]
# }
def read_metadata_file(s3_client, json_bucket, json_path):
    json_metadata = s3_client.get_object(Bucket=json_bucket, Key=json_path)
    json_contents = json.loads(json_metadata['Body'].read().decode('utf-8'))
    return json_contents

    # with open(path_to_json, 'r', encoding="utf8") as file:
    # json_file = json.load(file)
    # return json_file


def check_listing_response_fill_list(page, response_container_type, retention_start_date):
    # Check and compare each object in the provided s3 objects page (param: page) if its 'LastModified' datetime
    # parameter is older than the specified retention start date. If yes - object key is added to a list.
    objects_list = []
    for obj in page[response_container_type]:
        key = obj['Key']
        ver = obj['VersionId']
        last_modified = obj['LastModified']
        if last_modified < retention_start_date:
            objects_list.append((key, ver))

    return objects_list


def delete_objects(s3_client, bucket_name, objects_list):
    # Divide the list to chunks of 1000 objects each, as delete_objects function accepts max 1000 per call
    # Use Python list comprehension
    chunks = [objects_list[x:x + 1000] for x in range(0, len(objects_list), 1000)]

    # Create a list of dictionary elements for each file in the sublist(chunk)
    for chunk in chunks:
        objects = []
        for keyId, verId in chunk:
            objects.append(
                {
                    'Key': keyId,
                    'VersionId': verId
                }
            )
        # Actual deletion of objects. Supports two modes for the response: verbose and quiet In quiet mode the response
        # includes only keys where the deletion failed. If success - no info returned.
        # Default mode is Verbose (info is returned for each object)
        s3_client.delete_objects(
            Bucket=bucket_name,
            Delete={
                'Objects': objects,
                'Quiet': True
            }
        )


def save_log_file(s3_resource, log_bucket, log_path, job_name, log_file_contents):
    save_ts = datetime.timestamp(datetime.now(timezone.utc))
    log_file_name = f"{log_path}{job_name}/{job_name}.{save_ts}"
    log_obj = s3_resource.Object(log_bucket, log_file_name)
    log_file_body = "\n".join(log_file_contents)
    log_obj.put(Body=log_file_body)
    return


def run():
    print("Starting job execution...")
    now_ts = datetime.now(timezone.utc)

    # Load Job Parameters
    args = getResolvedOptions(sys.argv, [
        'ENV',
        'JOB',
        'JSON_BUCKET',
        'JSON_FILE_PATH',
        'LOG_BUCKET',
        'LOG_PATH'
    ])

    print("Parsed Job args: ")
    for k, v in args.items():
        print(f"{k}: {v}")

    job_env = args['ENV']
    job_name = args['JOB']
    json_metadata_bucket = args['JSON_BUCKET']
    json_metadata_file_path = args['JSON_FILE_PATH']
    log_bucket = args['LOG_BUCKET']
    log_path = check_log_path_add_slash(args['LOG_PATH'])

    s3_resource = boto3.resource('s3')
    s3_client = boto3.client('s3', region_name='eu-west-1')
    paginator = s3_client.get_paginator('list_object_versions')

    error_count = 0
    log_data = initiate_logger(now_ts, job_name, job_env)
    deleter_entities = read_metadata_file(s3_client, json_metadata_bucket, json_metadata_file_path)

    # Max Retention period is 5 years.
    # For each entity in metadata file, the respective location (and its subfolders, if any) are iterated and files with
    # LastModified date, older than specified retention_start_date are collected in a list, together with their
    # VersionIds. Then list is passed to the deletion function which deletes the file and its particular version, thus
    # file is permanently deleted, no previous version and delete marker are created (for s3 buckets with versioning)
    # Also a check for any leftover Delete markers older than retention_start_date is done - if any are found, they
    # are also to be deleted.
    for entity in deleter_entities["deleter"]:
        if job_env.upper() == ((entity["environment"]).upper()):
            try:
                project = entity["project"]
                bucket = entity["bucket"]
                prefix = entity["path_to_delete"]
                retain_days = entity["retain_days"]

                # Note: Timedelta calculation is end-date exclusive
                retention_start_date = now_ts - timedelta(days=retain_days)

                print(f"Bucket: {bucket} \nPrefix: {prefix} \nRetention Start Date: {retention_start_date}")
                log_data.append(f"Bucket: {bucket}")
                log_data.append(f"Prefix: {prefix}")
                log_data.append(f"Retention Start Date: {retention_start_date}. Removing all earlier files.")

                page_iterator = paginator.paginate(Bucket=bucket, Prefix=prefix)
                deleted_objects_list = []
                delete_markers_list = []
                for page in page_iterator:
                    # Collecting list of files/objects to be deleted
                    if 'Versions' in page:
                        deleted_objects_list.extend(check_listing_response_fill_list(page, 'Versions', retention_start_date))

                    # Collecting the VersionIds of Delete markers, so that they get wiped out too.
                    if 'DeleteMarkers' in page:
                        delete_markers_list.extend(check_listing_response_fill_list(page, 'DeleteMarkers', retention_start_date))

                deleted_objects_counter = len(deleted_objects_list)
                delete_markers_counter = len(delete_markers_list)
                for obj_list in (deleted_objects_list, delete_markers_list):
                    delete_objects(s3_client, bucket_name=bucket, objects_list=obj_list)

                print(f"Deleted objects count: {deleted_objects_counter}")
                print(f"Delete markers count: {delete_markers_counter}\n")
                log_data.append(f"{deleted_objects_counter} objects successfully deleted from location.")
                log_data.append(f"{delete_markers_counter} delete markers successfully deleted from location.")
                log_data.append(f"******")
            except Exception as ex:
                print(f"EXCEPTION RAISED: {ex}")
                log_data.append(f"PROCESSING FOR ENTITY FAILED!")
                log_data.append(f"ENV: {job_env} Bucket: {entity['bucket']} Path: {entity['path_to_delete']}")
                log_data.append(f"EXCEPTION RAISED: {ex}")
                error_count += 1
        else:
            not_matched_message = f"Environment not matched for entry: ENV: {entity['environment']}, " \
                                  f"Project: {entity['project']}, bucket: {entity['bucket']}, path: {entity['path_to_delete']}\n******\n"
            print(not_matched_message)
            log_data.append(not_matched_message)

    # Log save location could be added to deleter entities metadata file, so that logs which are older than max
    # retention period (e.g. ~5 years (1750 days)) are also deleted
    save_log_file(s3_resource, log_bucket, log_path, job_name, log_data)

    # Errors processing: fail job on purpose if error_count > 0
    print(f"Number of failed operations: {str(error_count)}")
    if error_count > 0:
        print("Errors found! Terminating job!")
        sys.exit()
    else:
        print("END!")


# Program entry point
if __name__ == '__main__':
    run()

# Function checks the path (string) provided as input param and adds a '/' if missing, as per AWS best practices # paths should end with a slash # Expected format of metadata json file: # { #     "version": "1.0", #     "deleter": [ #         { #             "environment": "PRD", #             "project": "My-Project", #             "bucket": "my-projects-bucket", #             "path_to_delete": "path/to/delete/", #             "retain_days": 1750 #         }, #         { #             ... #         } #     ] # } # with open(path_to_json, 'r', encoding="utf8") as file: # json_file = json.load(file) # return json_file # Check and compare each object in the provided s3 objects page (param: page) if its 'LastModified' datetime # parameter is older than the specified retention start date. If yes - object key is added to a list. # Divide the list to chunks of 1000 objects each, as delete_objects function accepts max 1000 per call # Use Python list comprehension # Create a list of dictionary elements for each file in the sublist(chunk) # Actual deletion of objects. Supports two modes for the response: verbose and quiet In quiet mode the response # includes only keys where the deletion failed. If success - no info returned. # Default mode is Verbose (info is returned for each object) # Load Job Parameters # Max Retention period is 5 years. # For each entity in metadata file, the respective location (and its subfolders, if any) are iterated and files with # LastModified date, older than specified retention_start_date are collected in a list, together with their # VersionIds. Then list is passed to the deletion function which deletes the file and its particular version, thus # file is permanently deleted, no previous version and delete marker are created (for s3 buckets with versioning) # Also a check for any leftover Delete markers older than retention_start_date is done - if any are found, they # are also to be deleted. # Note: Timedelta calculation is end-date exclusive # Collecting list of files/objects to be deleted # Collecting the VersionIds of Delete markers, so that they get wiped out too. # Log save location could be added to deleter entities metadata file, so that logs which are older than max # retention period (e.g. ~5 years (1750 days)) are also deleted # Errors processing: fail job on purpose if error_count > 0 # Program entry point