# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4; coding:utf-8 -*-
#
# Copyright 2002 Ben Escoto
# Copyright 2007 Kenneth Loafman
# Copyright 2019 Carl A. Adams
#
# This file is part of duplicity.
#
# Duplicity is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# Duplicity is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

"""
S3 Backend using boto3.
"""

import os

import duplicity.backend
from duplicity import config
from duplicity import file_naming
from duplicity import errors
from duplicity import log
from duplicity import progress
from duplicity.errors import BackendException
from duplicity.errors import FatalBackendException

global boto3, botocore, ClientError, S3UploadFailedError, TransferConfig


# Note: current gaps with the old boto backend include:
#       - Glacier restore to S3 not implemented. Should this
#         be done here? Or is that out of scope. My current opinion
#         is that it is out of scope, and the manpage reflects this.
#         It can take days, so waiting seems like it's not ideal.
#         "Thaw" isn't currently a generic concept that the core asks
#         of back-ends. Perhaps that is worth exploring.  The older
#         boto backend appeared  to attempt this restore in the code,
#         but the man page indicated that restores should be done out
#         of band. If implemented,  We should add the the following
#         new features:
#              - when restoring from glacier or deep archive, specify TTL.
#              - allow user to specify how fast to restore (impacts cost).


class S3Boto3Backend(duplicity.backend.Backend):
    """
    Backend for Amazon's Simple Storage System, (aka Amazon S3), though
    the use of the boto3 module. (See
    https://boto3.amazonaws.com/v1/documentation/api/latest/index.html
    for information on boto3.)

    Pursuant to Amazon's announced deprecation of path style S3 access,
    this backend only supports virtual host style bucket URIs.
    See the man page for full details.

    To make use of this backend, you must provide AWS credentials.
    This may be done in several ways: through the environment variables
    AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, by the
    ~/.aws/credentials file, by the ~/.aws/config file,
    or by using the boto2 style ~/.boto or /etc/boto.cfg files.
    """

    def __init__(self, parsed_url):
        """
        Initialize the S3 boto3 backend.

        :param ParsedUrl parsed_url: The parsed URL for the backend.
        :raises BackendException: If the bucket name is missing.
        """
        global boto3, botocore, ClientError, S3UploadFailedError, TransferConfig
        import boto3
        import botocore
        from botocore.exceptions import ClientError
        from boto3.exceptions import S3UploadFailedError
        from boto3.s3.transfer import TransferConfig

        duplicity.backend.Backend.__init__(self, parsed_url)

        # This folds the null prefix and all null parts, which means that:
        #  //MyBucket/ and //MyBucket are equivalent.
        #  //MyBucket//My///My/Prefix/ and //MyBucket/My/Prefix are equivalent.
        url_path_parts = [x for x in parsed_url.path.split("/") if x != ""]
        if url_path_parts:
            self.bucket_name = url_path_parts.pop(0)
        else:
            raise BackendException("S3 requires a bucket name.")

        if url_path_parts:
            self.key_prefix = f"{'/'.join(url_path_parts)}/"
        else:
            self.key_prefix = ""

        self.parsed_url = parsed_url
        self.straight_url = duplicity.backend.strip_auth_from_url(parsed_url)
        self.s3 = None
        self.bucket = None
        self.tracker = UploadProgressTracker()

        if not (boto3.__version__ < "1.36.0" and botocore.__version__ < "1.36.0"):
            # this is an issue with 3rd party s3 implementations only
            # likely when an endpoint is given that resides not under amazonaws.com
            # in time that workaround will probably not be needed anymore
            # https://github.com/boto/boto3/issues/2913
            import re

            if config.s3_endpoint_url and not re.match(
                pattern="(?i).*\\.amazonaws\\.com(/+)?$", string=config.s3_endpoint_url
            ):
                log.Notice(
                    "WARNING: If boto3 >= 1,36.0 with non-amazon s3 services"
                    " results in checksum errors preventing uploads you may try"
                    " the following workaround. set the following env vars\n\n"
                    "    export AWS_REQUEST_CHECKSUM_CALCULATION=when_required\n"
                    "    export AWS_RESPONSE_CHECKSUM_VALIDATION=when_required\n\n"
                    "see https://gitlab.com/duplicity/duplicity/-/issues/870 for details."
                )

    def reset_connection(self):
        """
        Reset the S3 connection.
        """
        self.bucket = None
        self.s3 = boto3.resource(
            "s3",
            region_name=config.s3_region_name,
            use_ssl=(not config.s3_unencrypted_connection),
            endpoint_url=config.s3_endpoint_url,
        )

        try:
            self.s3.meta.client.head_bucket(Bucket=self.bucket_name)
        except botocore.exceptions.ClientError as bce:
            error_code = bce.response["Error"]["Code"]
            if error_code == "404":
                raise FatalBackendException(
                    f'S3 bucket "{self.bucket_name}" does not exist',
                    code=log.ErrorCode.backend_not_found,
                )
            else:
                raise

        self.bucket = self.s3.Bucket(self.bucket_name)  # only set if bucket is thought to exist.

    def _put(self, local_source_path, remote_filename):
        """
        Copy a local file to the remote S3 bucket.
        """
        if not self.s3:
            self.reset_connection()

        # files that should not in glacier and deep_archive, to allow smooth operation
        glacier_exceptions = [
            file_naming.full_manifest_re,
            file_naming.inc_manifest_re,
            file_naming.full_sig_re,
            file_naming.new_sig_re,
        ]

        def is_glacier_exception(filename):
            return any([x.match(filename) for x in glacier_exceptions])

        if config.s3_use_rrs:
            storage_class = "REDUCED_REDUNDANCY"
        elif config.s3_use_ia:
            storage_class = "STANDARD_IA"
        elif config.s3_use_onezone_ia:
            storage_class = "ONEZONE_IA"
        elif config.s3_use_glacier and not is_glacier_exception(remote_filename):
            storage_class = "GLACIER"
        elif config.s3_use_glacier_ir and not is_glacier_exception(remote_filename):
            storage_class = "GLACIER_IR"
        elif config.s3_use_deep_archive and not is_glacier_exception(remote_filename):
            storage_class = "DEEP_ARCHIVE"
        else:
            storage_class = "STANDARD"
        extra_args = {"StorageClass": storage_class}

        if config.s3_use_sse:
            extra_args["ServerSideEncryption"] = "AES256"
        elif config.s3_use_sse_kms:
            if config.s3_kms_key_id is None:
                raise FatalBackendException(
                    "S3 USE SSE KMS was requested, but key id not provided " "require (--s3-kms-key-id)",
                    code=log.ErrorCode.s3_kms_no_id,
                )
            extra_args["ServerSideEncryption"] = "aws:kms"
            extra_args["SSEKMSKeyId"] = config.s3_kms_key_id
            if config.s3_kms_grant:
                extra_args["GrantFullControl"] = config.s3_kms_grant

        transfer_config = TransferConfig(
            multipart_chunksize=config.s3_multipart_chunk_size,
            multipart_threshold=config.s3_multipart_chunk_size,
            max_concurrency=config.s3_multipart_max_procs,
        )

        # Should the tracker be scoped to the put or the backend?
        # The put seems right to me, but the results look a little more correct
        # scoped to the backend.  This brings up questions about knowing when
        # it's proper for it to be reset.
        # tracker = UploadProgressTracker() # Scope the tracker to the put()
        tracker = self.tracker

        remote_filename = os.fsdecode(remote_filename)
        key = self.key_prefix + remote_filename

        log.Info(f"Uploading {self.straight_url}/{remote_filename} to {storage_class} Storage")
        try:
            self.s3.Object(self.bucket.name, key).upload_file(
                local_source_path.uc_name,
                Callback=tracker.progress_cb,
                Config=transfer_config,
                ExtraArgs=extra_args,
            )
        except S3UploadFailedError as e:
            if boto3.__version__ > "1.36.0" or botocore.__version__ > "1.36.0":
                raise FatalBackendException(
                    f"Failed to upload file {remote_filename}, got S3UploadFailedError\n"
                    f"See https://gitlab.com/duplicity/duplicity/-/issues/870 for details.\n"
                    f"Quick fix is: [sudo] pip3 install boto3<1.36.0 botocore<1.36.0\n"
                    f"API-Error: {e}"
                )
            else:
                raise e

    def _get(self, remote_filename, local_path):
        """
        Copy a remote S3 object to a local path.

        :param str remote_filename: The name of the file on the backend.
        :param Path local_path: The local path to copy the file to.
        :raises FatalError: If the file is in long term storage (Glacier).
        """
        if not self.s3:
            self.reset_connection()

        remote_filename = os.fsdecode(remote_filename)
        key = self.key_prefix + remote_filename
        try:
            self.s3.Object(self.bucket.name, key).download_file(local_path.uc_name)
        except ClientError as ios:
            if ios.response["Error"]["Code"] == "InvalidObjectState":
                raise errors.FatalError(
                    f"File {remote_filename} seems to be in a long term storage,"
                    f"Please use AWS Console/API to initiate restore.\n"
                    f"API-Error: {ios}"
                )
            else:
                raise ios

    def _list(self):
        """
        List objects in the S3 bucket under the key prefix.

        :returns: A list of filenames as byte strings.
        :rtype: list
        """
        if not self.s3:
            self.reset_connection()

        filename_list = []
        for obj in self.bucket.objects.filter(Prefix=self.key_prefix):
            try:
                filename = obj.key.replace(self.key_prefix, "", 1)
                filename_list.append(os.fsencode(filename))
                log.Debug(f"Listed {self.straight_url}/{filename}")
            except AttributeError:
                pass
        return filename_list

    def _delete(self, remote_filename):
        """
        Delete an object from the S3 bucket.

        :param str remote_filename: The name of the file to delete.
        """
        if not self.s3:
            self.reset_connection()

        remote_filename = os.fsdecode(remote_filename)
        key = self.key_prefix + remote_filename
        self.s3.Object(self.bucket.name, key).delete()

    def _query(self, remote_filename):
        """
        Query metadata for an object in the S3 bucket.

        :param str remote_filename: The name of the object to query.
        :returns: A dictionary containing metadata (e.g., 'size').
        :rtype: dict
        """
        if not self.s3:
            self.reset_connection()

        remote_filename = os.fsdecode(remote_filename)
        key = self.key_prefix + remote_filename
        content_length = -1
        try:
            s3_obj = self.s3.Object(self.bucket.name, key)
            s3_obj.load()
            content_length = s3_obj.content_length
        except botocore.exceptions.ClientError as bce:
            if bce.response["Error"]["Code"] == "404":
                pass
            else:
                raise
        return {"size": content_length}


class UploadProgressTracker(object):
    """
    Track progress of an upload.
    """

    def __init__(self):
        """
        Initialize the upload progress tracker.
        """
        self.total_bytes = 0

    def progress_cb(self, fresh_byte_count):
        """
        Callback for boto3 to report progress.
        """
        self.total_bytes += fresh_byte_count
        progress.report_transfer(self.total_bytes, 0)  # second arg appears to be unused
        # It would seem to me that summing progress should be the callers job,
        # and backends should just toss bytes written numbers over the fence.
        # But, the progress bar doesn't work in a reasonable way when we do
        # that. (This would also eliminate the need for this class to hold
        # the scoped rolling total.)
        # progress.report_transfer(fresh_byte_count, 0)


duplicity.backend.register_backend("boto3+s3", S3Boto3Backend)
# make boto3 the default s3 backend
duplicity.backend.register_backend("s3", S3Boto3Backend)
