        if not isinstance(request, Request):
            # Don't deep copy the file data as it may contain an open file handle
            old_file_data = copy.copy(request.FILES)
            old_post_data = copy.deepcopy(request.POST)
            request = super(ResourceFileCRUD, self).initialize_request(request, *args, **kwargs)
            request.POST._mutable = True
            request.POST.update(old_post_data)
            request.FILES.update(old_file_data)
        return request

import os
import mimetypes
import copy
import tempfile
import shutil
import logging
import json

from django.core.urlresolvers import reverse
from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation
from django.core.exceptions import ValidationError as CoreValidationError
from django.http import HttpResponseRedirect
from django.shortcuts import redirect
from django.contrib.sites.models import Site

from rest_framework.pagination import PageNumberPagination
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import generics, status
from rest_framework.request import Request
from rest_framework.exceptions import ValidationError, NotAuthenticated, PermissionDenied, NotFound

from hs_core import hydroshare
from hs_core.models import AbstractResource
from hs_core.hydroshare.utils import get_resource_by_shortkey, get_resource_types, \
    get_content_types
from hs_core.views import utils as view_utils
from hs_core.views.utils import ACTION_TO_AUTHORIZE
from hs_core.views import serializers
from hs_core.hydroshare.utils import get_file_storage, resource_modified
from hs_core.serialization import GenericResourceMeta, HsDeserializationDependencyException, \
    HsDeserializationException
from hs_core.hydroshare.hs_bagit import create_bag_metadata_files
from drf_yasg.utils import swagger_auto_schema
from rest_framework.parsers import MultiPartParser

logger = logging.getLogger(__name__)

# Mixins
class ResourceFileToListItemMixin(object):
    def resourceFileToListItem(self, f):
        # URLs in metadata should be fully qualified.
        # ALWAYS qualify them with www.hydroshare.org, rather than the local server name.
        site_url = hydroshare.utils.current_site_url()
        url = site_url + f.url
        fsize = f.size
        logical_file_type = f.logical_file_type_name
        file_name = os.path.basename(f.resource_file.name)
        modified_time = f.modified_time
        checksum = f.checksum
        # trailing slash confuses mime guesser
        mimetype = mimetypes.guess_type(url)
        if mimetype[0]:
            ftype = mimetype[0]
        else:
            ftype = repr(None)
        resource_file_info_item = serializers.ResourceFileItem(url=url,
                                                               file_name=file_name,
                                                               size=fsize,
                                                               content_type=ftype,
                                                               logical_file_type=logical_file_type,
                                                               modified_time=modified_time,
                                                               checksum=checksum)
        return resource_file_info_item

class ResourceTypes(generics.ListAPIView):
    # We don't need pagination for a list of resource types
    pagination_class = None

    @swagger_auto_schema(operation_description="List Resource Types",
                         responses={200: serializers.ResourceTypesSerializer})
    def get(self, request):
        return self.list(request)

    def get_queryset(self):
        return [serializers.ResourceType(resource_type=rtype.__name__) for rtype in
                get_resource_types()]

    def get_serializer_class(self):
        return serializers.ResourceTypesSerializer

class ContentTypes(generics.ListAPIView):
    pagination_class = None

    @swagger_auto_schema(operation_description="List Content Types",
                         responses={200: serializers.ContentTypesSerializer})
    def get(self, request):
        return self.list(request)

    def get_queryset(self):
        return [serializers.ContentType(content_type=ctype.__name__) for ctype in
                get_content_types()]

    def get_serializer_class(self):
        return serializers.ContentTypesSerializer

class CheckTaskStatus(generics.RetrieveAPIView):

    # TODO, setup a serializer for in/out, figure out if redirect is needed...
    def get(self, request, task_id):
        url = reverse('rest_check_task_status', kwargs={'task_id': task_id})
        return HttpResponseRedirect(url)

class ResourceReadUpdateDelete(generics.RetrieveUpdateDestroyAPIView):
    # pagination doesn't make sense as there is only one resource
    pagination_class = None

    allowed_methods = ('GET', 'PUT', 'DELETE')

    @swagger_auto_schema(operation_description="Get a resource in zipped BagIt format",
                         responses={200: serializers.TaskStatusSerializer})
    def get(self, request, pk):
        res, _, _ = view_utils.authorize(request, pk,
                                         needed_permission=ACTION_TO_AUTHORIZE.VIEW_RESOURCE)
        if res.resource_type.lower() == "reftimeseriesresource":

            bag_url = reverse('rest_download_refts_resource_bag',
                              kwargs={'shortkey': pk})
        else:
            bag_url = reverse('rest_download',
                              kwargs={'path': 'bags/{}.zip'.format(pk)})
        return HttpResponseRedirect(bag_url)

    @swagger_auto_schema(operation_description="Not Implemented")
    def put(self, request, pk):
        # TODO: update resource - involves overwriting a resource from the provided bag file
        raise NotImplementedError()

    def delete(self, request, pk):
        # only resource owners are allowed to delete
        view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.DELETE_RESOURCE)
        hydroshare.delete_resource(pk, request_username=request.user.username)
        return Response(status=status.HTTP_204_NO_CONTENT)

class ResourceListCreate(generics.ListCreateAPIView):

    @swagger_auto_schema(request_body=serializers.ResourceCreateRequestValidator,
                         operation_description="Create a resource",
                         responses={201: serializers.ResourceCreatedSerializer})
    def post(self, request):
        return self.create(request)

    # Override the create() method from the CreateAPIView class
    def create(self, request, *args, **kwargs):
        if not request.user.is_authenticated():
            raise NotAuthenticated()

        resource_create_request_validator = serializers.ResourceCreateRequestValidator(
            data=request.data)
        if not resource_create_request_validator.is_valid():
            raise ValidationError(detail=resource_create_request_validator.errors)

        validated_request_data = resource_create_request_validator.validated_data
        resource_type = validated_request_data['resource_type']
        if resource_type in ["RasterResource", "GeographicFeatureResource", "RefTimeSeriesResource",
                             "NetcdfResource", "TimeSeriesResource", "GenericResource"]:
            # force deprecated resource types to composite
            resource_type = "CompositeResource"

        res_title = validated_request_data.get('title', 'Untitled resource')
        keywords = validated_request_data.get('keywords', [])
        abstract = validated_request_data.get('abstract', None)
        metadata = validated_request_data.get('metadata', None)
        extra_metadata = validated_request_data.get('extra_metadata', None)

        num_files = len(request.FILES)
        # TODO: (Couch) reconsider whether multiple file upload should be
        # supported when multipart bug fixed.
        if num_files > 0:
            if num_files > 1:
                raise ValidationError(detail={'file': 'Multiple file upload is not allowed on '
                                                      'resource creation. Add additional files '
                                                      'after the resource is created.'})
            # Place files into format expected by hydroshare.utils.resource_pre_create_actions and
            # hydroshare.create_resource, i.e. a tuple of
            # django.core.files.uploadedfile.TemporaryUploadedFile objects.
            files = [request.FILES['file'], ]
        else:
            files = []

        if metadata is not None:
            metadata = json.loads(metadata)
            _validate_metadata(metadata)

        if extra_metadata is not None:
            extra_metadata = json.loads(extra_metadata)
            # TODO: validate extra metadata here

        try:
            _, res_title, metadata = hydroshare.utils.resource_pre_create_actions(
                resource_type=resource_type, resource_title=res_title,
                page_redirect_url_key=None, files=files, metadata=metadata,
                **kwargs)
        except Exception as ex:
            error_msg = {'resource': "Resource creation failed. %s" % str(ex)}
            raise ValidationError(detail=error_msg)

        try:
            resource = hydroshare.create_resource(
                resource_type=resource_type,
                owner=request.user,
                title=res_title,
                edit_users=validated_request_data.get('edit_users', None),
                view_users=validated_request_data.get('view_users', None),
                edit_groups=validated_request_data.get('edit_groups', None),
                view_groups=validated_request_data.get('view_groups', None),
                keywords=keywords,
                metadata=metadata,
                extra_metadata=extra_metadata,
                files=files
            )
            if abstract:
                resource.metadata.create_element('description', abstract=abstract)
        except Exception as ex:
            error_msg = {'resource': "Resource creation failed. %s" % str(ex)}
            raise ValidationError(detail=error_msg)

        post_creation_error_msg = ''
        try:
            hydroshare.utils.resource_post_create_actions(request=request, resource=resource,
                                                          user=request.user,
                                                          metadata=metadata, **kwargs)
        except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
            post_creation_error_msg = str(ex)
        response_data = {'resource_type': resource_type, 'resource_id': resource.short_id,
                         'message': post_creation_error_msg}

        return Response(data=response_data,  status=status.HTTP_201_CREATED)

    pagination_class = PageNumberPagination
    pagination_class.page_size_query_param = 'count'

    @swagger_auto_schema(query_serializer=serializers.ResourceListRequestValidator,
                         operation_description="List resources")
    def get(self, request):
        return self.list(request)

    # needed for list of resources
    # copied from ResourceList
    def get_queryset(self):
        resource_list_request_validator = serializers.ResourceListRequestValidator(
            data=self.request.query_params)
        if not resource_list_request_validator.is_valid():
            raise ValidationError(detail=resource_list_request_validator.errors)

        filter_parms = resource_list_request_validator.validated_data
        filter_parms['user'] = (self.request.user if self.request.user.is_authenticated() else None)
        if len(filter_parms['type']) == 0:
            filter_parms['type'] = None
        else:
            filter_parms['type'] = list(filter_parms['type'])

        filter_parms['public'] = not self.request.user.is_authenticated()

        return hydroshare.get_resource_list(**filter_parms)

    # covers serialization of output from GET request
    def get_serializer_class(self):
        return serializers.ResourceListItemSerializer

    # covers serialization of output from POST request
    def post_serializer_class(self):
        return serializers.ResourceCreatedSerializer

class SystemMetadataRetrieve(APIView):

    allowed_methods = ('GET',)

    @swagger_auto_schema(operation_description="Get resource system metadata, as well as URLs to "
                                               "the bag and science metadata",
                         responses={200: serializers.ResourceListItemSerializer})
    def get(self, request, pk):
        res, _, _ = view_utils.authorize(request, pk,
                                         needed_permission=ACTION_TO_AUTHORIZE.VIEW_METADATA)
        ser = self.get_serializer_class()(res)

        return Response(data=ser.data, status=status.HTTP_200_OK)

    def get_serializer_class(self):
        return serializers.ResourceListItemSerializer

class AccessRulesUpdate(APIView):
    """
    Set access rules for a resource

    REST URL: hsapi/resource/{pk}/access
    DEPRECATED: hsapi/resource/accessRules/{pk}
    HTTP method: PUT

    :type pk: str
    :param pk: id of the resource
    :return: No content.  Status code will 200 (OK)
    """
    allowed_methods = ('PUT',)

    def put(self, request, pk):
        """ Update access rules
        """
        # only resource owners are allowed to change resource flags (e.g., public)
        view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.SET_RESOURCE_FLAG)

        access_rules_validator = serializers.AccessRulesRequestValidator(data=request.data)
        if not access_rules_validator.is_valid():
            raise ValidationError(detail=access_rules_validator.errors)

        validated_request_data = access_rules_validator.validated_data
        res = get_resource_by_shortkey(pk)
        try:
            res.set_public(validated_request_data['public'], request.user)
        except CoreValidationError:
            return Response(data={'resource_id': pk}, status=status.HTTP_403_FORBIDDEN)

        return Response(data={'resource_id': pk}, status=status.HTTP_200_OK)

class ScienceMetadataRetrieveUpdate(APIView):
    """
    Retrieve resource science metadata

    REST URL: hsapi/scimeta/{pk}
    HTTP method: GET

    :type pk: str
    :param pk: id of the resource
    :return: science metadata as XML document
    :rtype: str
    :raises:
    NotFound: return json format: {'detail': 'No resource was found for resource id:pk'}
    PermissionDenied: return json format: {'detail': 'You do not have permission to perform
    this action.'}

    REST URL: hsapi/scimeta/{pk}
    HTTP method: PUT

    :type pk: str
    :param pk: id of the resource
    :type metadata: json
    :param metadata: resource metadata
    :return: resource id
    :rtype: json of the format: {'resource_id':pk}
    :raises:
    NotFound: return json format: {'detail': 'No resource was found for resource id':pk}
    PermissionDenied: return json format: {'detail': 'You do not have permission to perform
    this action.'}
    ValidationError: return json format: {parameter-1': ['error message-1'],
    'parameter-2': ['error message-2'], .. }
    """
    ACCEPT_FORMATS = ('application/xml', 'application/rdf+xml')

    allowed_methods = ('GET', 'PUT')

    def get(self, request, pk):
        view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.VIEW_METADATA)

        scimeta_url = AbstractResource.scimeta_url(pk)
        return redirect(scimeta_url)

    def put(self, request, pk):
        # Update science metadata based on resourcemetadata.xml uploaded
        resource, authorized, user = view_utils.authorize(
            request, pk,
            needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE,
            raises_exception=False)
        if not authorized:
            raise PermissionDenied()

        files = list(request.FILES.values())
        if len(files) == 0:
            error_msg = {'file': 'No resourcemetadata.xml file was found to update resource '
                                 'metadata.'}
            raise ValidationError(detail=error_msg)
        elif len(files) > 1:
            error_msg = {'file': ('More than one file was found. Only one file, named '
                                  'resourcemetadata.xml, '
                                  'can be used to update resource metadata.')}
            raise ValidationError(detail=error_msg)

        scimeta = files[0]
        if scimeta.content_type not in self.ACCEPT_FORMATS:
            error_msg = {'file': ("Uploaded file has content type {t}, "
                                  "but only these types are accepted: {e}.").format(
                t=scimeta.content_type, e=",".join(self.ACCEPT_FORMATS))}
            raise ValidationError(detail=error_msg)
        expect = 'resourcemetadata.xml'
        if scimeta.name != expect:
            error_msg = {'file': "Uploaded file has name {n}, but expected {e}.".format(
                n=scimeta.name, e=expect)}
            raise ValidationError(detail=error_msg)

        # Temp directory to store resourcemetadata.xml
        tmp_dir = tempfile.mkdtemp()
        try:
            # Fake the bag structure so that GenericResourceMeta.read_metadata_from_resource_bag
            # can read and validate the system and science metadata for us.
            bag_data_path = os.path.join(tmp_dir, 'data')
            os.mkdir(bag_data_path)
            # Copy new science metadata to bag data path
            scimeta_path = os.path.join(bag_data_path, 'resourcemetadata.xml')
            shutil.copy(scimeta.temporary_file_path(), scimeta_path)
            # Copy existing resource map to bag data path
            # (use a file-like object as the file may be in iRODS, so we can't
            #  just copy it to a local path)
            resmeta_path = os.path.join(bag_data_path, 'resourcemap.xml')
            with open(resmeta_path, 'wb') as resmeta:
                storage = get_file_storage()
                resmeta_irods = storage.open(AbstractResource.sysmeta_path(pk))
                shutil.copyfileobj(resmeta_irods, resmeta)

            resmeta_irods.close()

            try:
                # Read resource system and science metadata
                domain = Site.objects.get_current().domain
                rm = GenericResourceMeta.read_metadata_from_resource_bag(tmp_dir,
                                                                         hydroshare_host=domain)
                # Update resource metadata
                rm.write_metadata_to_resource(resource, update_title=True, update_keywords=True)
                create_bag_metadata_files(resource)
            except HsDeserializationDependencyException as e:
                msg = ("HsDeserializationDependencyException encountered when updating "
                       "science metadata for resource {pk}; depedent resource was {dep}.")
                msg = msg.format(pk=pk, dep=e.dependency_resource_id)
                logger.error(msg)
                raise ValidationError(detail=msg)
            except HsDeserializationException as e:
                raise ValidationError(detail=str(e))

            resource_modified(resource, request.user, overwrite_bag=False)
            return Response(data={'resource_id': pk}, status=status.HTTP_202_ACCEPTED)
        finally:
            shutil.rmtree(tmp_dir)

class ResourceMapRetrieve(APIView):
    """
    Retrieve resource map

    REST URL: hsapi/resource/{pk}/map
    HTTP method: GET

    :type pk: str
    :param pk: id of the resource
    :return: resource map as XML document
    :rtype: str
    :raises:
    NotFound: return json format: {'detail': 'No resource was found for resource id:pk'}
    PermissionDenied: return json format: {'detail': 'You do not have permission to perform
    this action.'}
    """
    allowed_methods = ('GET',)

    def get(self, request, pk):
        view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.VIEW_METADATA)

        resmap_url = AbstractResource.resmap_url(pk)
        return redirect(resmap_url)

class ResourceFileCRUD(APIView):
    """
    Retrieve, add, update or delete a resource file

    REST URL: hsapi/resource/{pk}/files/{filename}
    HTTP method: GET

    :type pk: str
    :type filename: str
    :param pk: resource id
    :param filename: name of the file to retrieve/download
    :return: resource file data
    :rtype: file data bytes

    REST URL: POST hsapi/resource/{pk}/files/
    UNUSED: See ResourceFileListCreate for details.
    HTTP method: POST

    Request post data: file data (required)
    :type pk: str
    :param pk: resource id
    :return: id of the resource and name of the file added
    :rtype: json string of format: {'resource_id':pk, 'file_name': name of the file added}

    REST URL: hsapi/resource/{pk}/files/{filename}
    HTTP method: PUT

    :type pk: str
    :type filename: str
    :param pk: resource id
    :param filename: name of the file to update
    :return: id of the resource and name of the file
    :rtype: json string of format: {'resource_id':pk, 'file_name': name of the file updates}

    REST URL: hsapi/resource/{pk}/files/{filename}
    HTTP method: DELETE

    :type pk: str
    :type filename: str
    :param pk: resource id
    :param filename: name of the file to delete
    :return: id of the resource and name of the file
    :rtype: json string of format: {'resource_id':pk, 'file_name': name of the file deleted}

    :raises:
    NotFound: return json format: {'detail': 'No resource was found for resource id':pk}
    PermissionDenied: return json format: {'detail': 'You do not have permission to perform
    this action.'}
    ValidationError: return json format: {'parameter-1':['error message-1'],
    'parameter-2': ['error message-2'], .. }
    """
    allowed_methods = ('GET', 'POST', 'PUT', 'DELETE')
    parser_classes = (MultiPartParser,)

    def initialize_request(self, request, *args, **kwargs):
        """
        Hack to work around the following issue in django-rest-framework:

        http://example.com

        Couch: This issue was recently closed (10/12/2016, 2 days before this writing)
        and is slated to be incorporated in the Django REST API 3.5.0 release.
        At that time, we should remove this hack.

        :param request:
        :param args:
        :param kwargs:
        :return:
        """

    def get(self, request, pk, pathname):
        resource, _, _ = view_utils.authorize(
            request, pk,
            needed_permission=ACTION_TO_AUTHORIZE.VIEW_RESOURCE)

        if not resource.supports_folders and '/' in pathname:
            return Response("Resource type does not support folders", status.HTTP_403_FORBIDDEN)

        try:
            view_utils.irods_path_is_allowed(pathname)
        except (ValidationError, SuspiciousFileOperation) as ex:
            return Response(str(ex), status_code=status.HTTP_400_BAD_REQUEST)

        try:
            f = hydroshare.get_resource_file(pk, pathname).resource_file
        except ObjectDoesNotExist:
            err_msg = 'File with file name {file_name} does not exist for resource with ' \
                      'resource id {res_id}'.format(file_name=pathname, res_id=pk)
            raise NotFound(detail=err_msg)

        # redirects to django_irods/views.download function
        # use new internal url for rest call
        # TODO: (Couch) Migrate model (with a "data migration") so that this hack is not needed.
        redirect_url = f.url.replace('django_irods/download/', 'django_irods/rest_download/')
        return HttpResponseRedirect(redirect_url)

    @swagger_auto_schema(request_body=serializers.ResourceFileValidator)
    def post(self, request, pk, pathname):
        """
        Add a file to a resource.
        :param request:
        :param pk: Primary key of the resource (i.e. resource short ID)
        :param pathname: the path to the containing folder in the folder hierarchy
        :return:

        Leaving out pathname in the URI calls a different class function in ResourceFileListCreate
        that stores in the root directory instead.
        """
        resource, _, _ = view_utils.authorize(request, pk,
                                              needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)

        resource_files = list(request.FILES.values())
        if len(resource_files) == 0:
            error_msg = {'file': 'No file was found to add to the resource.'}
            raise ValidationError(detail=error_msg)
        elif len(resource_files) > 1:
            error_msg = {'file': 'More than one file was found. Only one file can be '
                                 'added at a time.'}
            raise ValidationError(detail=error_msg)

        # TODO: (Brian) I know there has been some discussion when to validate a file
        # I agree that we should not validate and extract metadata as part of the file add api
        # Once we have a decision, I will change this implementation accordingly. In that case
        # we have to implement additional rest endpoints for file validation and extraction.
        try:
            hydroshare.utils.resource_file_add_pre_process(resource=resource,
                                                           files=[resource_files[0]],
                                                           folder=pathname,
                                                           user=request.user, extract_metadata=True)

        except (hydroshare.utils.ResourceFileSizeException,
                hydroshare.utils.ResourceFileValidationException, Exception) as ex:
            error_msg = {'file': 'Adding file to resource failed. %s' % str(ex)}
            raise ValidationError(detail=error_msg)

        try:
            res_file_objects = hydroshare.utils.resource_file_add_process(resource=resource,
                                                                          files=[resource_files[0]],
                                                                          folder=pathname,
                                                                          user=request.user,
                                                                          extract_metadata=True)

        except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
            error_msg = {'file': 'Adding file to resource failed. %s' % str(ex)}
            raise ValidationError(detail=error_msg)

        # prepare response data
        file_name = os.path.basename(res_file_objects[0].resource_file.name)
        file_path = res_file_objects[0].resource_file.name.split('/data/contents/')[1]
        response_data = {'resource_id': pk, 'file_name': file_name, 'file_path': file_path}
        resource_modified(resource, request.user, overwrite_bag=False)
        return Response(data=response_data, status=status.HTTP_201_CREATED)

    def delete(self, request, pk, pathname):
        resource, _, user = view_utils.authorize(
            request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)

        if not resource.supports_folders and '/' in pathname:
            return Response("Resource type does not support folders", status.HTTP_403_FORBIDDEN)

        try:
            view_utils.irods_path_is_allowed(pathname)  # check for hacking attempts
        except (ValidationError, SuspiciousFileOperation) as ex:
            return Response(str(ex), status=status.HTTP_400_BAD_REQUEST)

        try:
            hydroshare.delete_resource_file(pk, pathname, user)
        except ObjectDoesNotExist as ex:    # matching file not found
            raise NotFound(detail=str(ex))

        # prepare response data
        response_data = {'resource_id': pk, 'file_name': pathname}
        resource_modified(resource, request.user, overwrite_bag=False)
        return Response(data=response_data, status=status.HTTP_200_OK)

    @swagger_auto_schema(auto_schema=None)
    def put(self, request, pk, pathname):
        # TODO: (Brian) Currently we do not have this action for the front end. Will implement
        # in the next iteration. Implement only after we have a decision on when to validate a file
        raise NotImplementedError()

class ResourceFileListCreate(ResourceFileToListItemMixin, generics.ListCreateAPIView):
    """
    Create a resource file or retrieve a list of resource files

    REST URL: hsapi/resource/{pk}/files/
    DEPRECATED: hsapi/resource/{pk}/file_list/
    HTTP method: GET

    :type pk: str
    :type filename: str
    :param pk: resource id
    :param filename: name of the file to retrieve/download
    :return: JSON representation of list of files of the form:

    REST URL: POST hsapi/resource/{pk}/files/
    HTTP method: POST

    Request post data: file data (required)
    :type pk: str
    :param pk: resource id
    :return: id of the resource and name of the file added
    :rtype: json string of format: {'resource_id':pk, 'file_name': name of the file added}

    {
        "count": 2,
        "next": null,
        "previous": null,
        "results": [
            {
                "url": "http://example.com
                download/bd88d2a152894134928c587d38cf0272/data/contents/
                mytest_resource/text_file.txt",
                "size": 21,
                "content_type": "text/plain",
                "modified_time": "2020-02-25T08:28:14",
                "checksum": "7265548b8f345605113bd9539313b4e7"
            },
            {
                "url": "http://example.com
                bd88d2a152894134928c587d38cf0272/data/contents/mytest_resource/a_directory/cea.tif",
                "size": 270993,
                "content_type": "image/tiff",
                "modified_time": "2020-02-25T08:28:14",
                "checksum": "ed06b456c22f7123d20888d16bcd181d"
            }
        ]
    }

    :raises:
    NotFound: return json format: {'detail': 'No resource was found for resource id':pk}
    PermissionDenied: return json format: {'detail': 'You do not have permission to perform
    this action.'}

    """
    allowed_methods = ('GET', 'POST')

    def initialize_request(self, request, *args, **kwargs):
        """
        Hack to work around the following issue in django-rest-framework:

        https://github.com/tomchristie/django-rest-framework/issues/3951

        Couch: This issue was recently closed (10/12/2016, 2 days before this writing)
        and is slated to be incorporated in the Django REST API 3.5.0 release.
        At that time, we should remove this hack.

        :param request:
        :param args:
        :param kwargs:
        :return:
        """
        if not isinstance(request, Request):
            # Don't deep copy the file data as it may contain an open file handle
            old_file_data = copy.copy(request.FILES)
            old_post_data = copy.deepcopy(request.POST)
            request = super(ResourceFileListCreate, self).initialize_request(
                request, *args, **kwargs)
            request.POST._mutable = True
            request.POST.update(old_post_data)
            request.FILES.update(old_file_data)
        return request

    def get(self, request, pk):
        """
        Get a listing of files within a resource.
        :param request:
        :param pk: Primary key of the resource (i.e. resource short ID)
        :return:
        """
        return self.list(request)

    def get_queryset(self):
        resource, _, _ = view_utils.authorize(self.request, self.kwargs['pk'],
                                              needed_permission=ACTION_TO_AUTHORIZE.VIEW_RESOURCE)
        resource_file_info_list = []
        for f in resource.files.all():
            resource_file_info_list.append(self.resourceFileToListItem(f))
        return resource_file_info_list

    def get_serializer_class(self):
        return serializers.ResourceFileSerializer

    def post(self, request, pk):
        """
        Add a file to a resource.
        :param request:
        :param pk: Primary key of the resource (i.e. resource short ID)
        :return:
        """
        resource, _, _ = view_utils.authorize(request, pk,
                                              needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
        resource_files = list(request.FILES.values())
        if len(resource_files) == 0:
            error_msg = {'file': 'No file was found to add to the resource.'}
            raise ValidationError(detail=error_msg)
        elif len(resource_files) > 1:
            error_msg = {'file': 'More than one file was found. Only one file can be '
                                 'added at a time.'}
            raise ValidationError(detail=error_msg)

        # TODO: (Brian) I know there has been some discussion when to validate a file
        # I agree that we should not validate and extract metadata as part of the file add api
        # Once we have a decision, I will change this implementation accordingly. In that case
        # we have to implement additional rest endpoints for file validation and extraction.
        folder = request.POST.get('folder', '')
        try:
            hydroshare.utils.resource_file_add_pre_process(resource=resource,
                                                           files=[resource_files[0]],
                                                           user=request.user,
                                                           folder=folder,
                                                           extract_metadata=True)

        except (hydroshare.utils.ResourceFileSizeException,
                hydroshare.utils.ResourceFileValidationException, Exception) as ex:
            error_msg = {'file': 'Adding file to resource failed. %s' % str(ex)}
            raise ValidationError(detail=error_msg)

        try:
            res_file_objects = hydroshare.utils.resource_file_add_process(resource=resource,
                                                                          files=[resource_files[0]],
                                                                          user=request.user,
                                                                          folder=folder,
                                                                          extract_metadata=True)

        except (hydroshare.utils.ResourceFileValidationException, Exception) as ex:
            error_msg = {'file': 'Adding file to resource failed. %s' % str(ex)}
            raise ValidationError(detail=error_msg)

        if len(res_file_objects) == 0:
            # metadata ingestion
            response_data = {'resource_id': pk}
        else:
            file_name = os.path.basename(res_file_objects[0].resource_file.name)
            file_path = res_file_objects[0].resource_file.name.split('/data/contents/')[1]
            response_data = {'resource_id': pk, 'file_name': file_name, 'file_path': file_path}
        resource_modified(resource, request.user, overwrite_bag=False)
        return Response(data=response_data, status=status.HTTP_201_CREATED)

def _validate_metadata(metadata_list):
    """
    Make sure the metadata_list does not have data for the following
    core metadata elements. Exception is raised if any of the following elements is present
    in metadata_list:

    title - (endpoint has a title parameter which should be used for specifying resource title)
    subject (keyword) - (endpoint has a keywords parameter which should be used for specifying
    resource keywords)
    description (abstract)- (endpoint has a abstract parameter which should be used for specifying
    resource abstract)
    publisher - this element is created upon resource publication
    format - this element is created by the system based on the resource content files
    date - this element is created by the system
    type - this element is created by the system

    :param metadata_list: list of dicts each representing data for a specific metadata element
    :return:
    """

    err_message = "Metadata validation failed. Metadata element '{}' was found in value passed " \
                  "for parameter 'metadata'. Though it's a valid element it can't be passed " \
                  "as part of 'metadata' parameter."
    for element in metadata_list:
        # here k is the name of the element
        # v is a dict of all element attributes/field names and field values
        k, v = list(element.items())[0]
        if k.lower() in ('title', 'subject', 'description', 'publisher', 'format', 'date', 'type'):
            err_message = err_message.format(k.lower())
            raise ValidationError(detail=err_message)


while i != -1:  # 前半区不停二分，直到找不到target为止
            start = i
            i = self.bSearch(A, 0, start, target)

class Solution:
    """
    @param A : a list of integers
    @param target : an integer to be searched
    @return : a list of length 2, [index1, index2]
    """
    def searchRange(self, A, target):
        # write your code here
        i = self.bSearch(A, 0, len(A), target)
        if i == -1:
            return [-1, -1]
        start, end = i, i

        if end < len(A) - 1:
            i = end
            while (i != -1) and (i <= (len(A) - 1)):
                end = i
                i = self.bSearch(A, end + 1, len(A), target)
                if i == -1:
                    break
        return [start, end]

    def bSearch(self, array, start, end, target):
        while start < end:
            mid = (start + end) / 2
            if array[mid] == target:
                return mid
            elif array[mid] < target:
                start = mid + 1
            else:
                end = mid   # end是开区间，所以不用减1。
        return -1


import numpy

class OrderSampler(object):

    """Base class of all order samplers.
    Every order sampler subclass has to provide a method
    :meth:`__call__`.
    This method is called by an iterator before a new epoch,
    and it should return a new index order for the next epoch.
    """

    def __call__(self, current_order, current_position):
        """Sample the next order.
        Args:
            current_order (numpy.ndarray): 1-D array of indices.
                The length should be the same as the dataset to sample
                data from.
            current_position (int): The current position of an iterator.
        Returns:
            numpy.ndarray:
            1-D array of indices. This is the order in which
            examples are sampled from a dataset in the next epoch.
        """

class ShuffleOrderSampler(OrderSampler):

    """Sampler that generates random orders.
    This is expected to be used together with Chainer's iterators.
    An order sampler is called by an iterator every epoch.
    The two initializations below create basically the same objects.
    >>> dataset = [(1, 2), (3, 4)]
    >>> it = chainer.iterators.MultiprocessIterator(dataset, 1, shuffle=True)
    >>> it = chainer.iterators.MultiprocessIterator(
    ...     dataset, 1, order_sampler=chainer.iterators.ShuffleOrderSampler())
    Args:
        random_state (numpy.random.RandomState): Pseudo-random number
            generator.
    """

    def __init__(self, random_state=None):
        if random_state is None:
            random_state = numpy.random.random.__self__
        self._random = random_state

    def __call__(self, current_order, current_position):
        return self._random.permutation(len(current_order))


while True:
            x = pyprob.sample(uniform)
            y = pyprob.sample(uniform)
            s = x*x + y*y
            i += 1
            if float(s) < 1:
                pyprob.tag(x, name='x_accepted')
                pyprob.tag(y, name='y_accepted')
                pyprob.tag(s, name='s_accepted')
                break
            else:
                pyprob.tag(x, name='x_rejected')
                pyprob.tag(y, name='y_rejected')
                pyprob.tag(s, name='s_rejected')

import torch
import math
import os
import shutil

import pyprob
import pyprob.diagnostics
from pyprob import Model, InferenceEngine
from pyprob.distributions import Uniform, Normal

class GaussianWithUnknownMeanMarsaglia(Model):
    def __init__(self, prior_mean=1, prior_stddev=math.sqrt(5), likelihood_stddev=math.sqrt(2), replace=True, *args, **kwargs):
        self.prior_mean = prior_mean
        self.prior_stddev = prior_stddev
        self.likelihood_stddev = likelihood_stddev
        self.replace = replace
        super().__init__('Gaussian with unknown mean (Marsaglia)', *args, **kwargs)

    def marsaglia(self, mean, stddev):
        uniform = Uniform(-1, 1)
        s = 1
        i = 0

        pyprob.tag(i, name='iterations')
        return mean + stddev * (x * torch.sqrt(-2 * torch.log(s) / s))

    def forward(self):
        mu = self.marsaglia(self.prior_mean, self.prior_stddev)
        likelihood = Normal(mu, self.likelihood_stddev)
        pyprob.tag(mu, name='mu')
        pyprob.observe(likelihood, name='obs0')
        pyprob.observe(likelihood, name='obs1')
        return mu

def produce_results(replace, results_dir):
    num_traces = 100000
    num_ic_training_traces = 1000000

    if os.path.exists(results_dir):
        shutil.rmtree(results_dir)
    pyprob.util.create_path(results_dir, directory=True)

    address_dict_file_name = os.path.join(results_dir, 'address_dict')
    model = GaussianWithUnknownMeanMarsaglia(address_dict_file_name=address_dict_file_name, replace=replace)

    ground_truth_trace = next(model._trace_generator(inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS))
    observes = {'obs0': ground_truth_trace.named_variables['obs0'].value, 'obs1': ground_truth_trace.named_variables['obs1'].value}

    posterior_is = model.posterior(num_traces, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING, observe=observes)
    proposal_is = posterior_is.unweighted().rename(posterior_is.name.replace('Posterior', 'Proposal'))

    model.learn_inference_network(num_ic_training_traces, observe_embeddings={'obs0': {}, 'obs1': {}}, inference_network=pyprob.InferenceNetwork.LSTM)
    posterior_ic = model.posterior(num_traces, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK, observe=observes)
    proposal_ic = posterior_ic.unweighted().rename(posterior_ic.name.replace('Posterior', 'Proposal'))

    posterior_rmh_file_name = os.path.join(results_dir, 'posterior_rmh')
    posterior_rmh = model.posterior(num_traces, inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS, observe=observes)

    posterior_rmh_autocorrelation_file_name = os.path.join(results_dir, 'posterior_rmh_autocorrelation')
    pyprob.diagnostics.autocorrelation(posterior_rmh, n_most_frequent=50, plot=True, plot_show=False, file_name=posterior_rmh_autocorrelation_file_name)

    posterior_rmh_gt_file_name = os.path.join(results_dir, 'posterior_rmh_gt')
    posterior_rmh_gt = model.posterior(num_traces, inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS, observe=observes, initial_trace=ground_truth_trace)

    posterior_rmh_gr_file_name = os.path.join(results_dir, 'posterior_rmh_gelman_rubin')
    pyprob.diagnostics.gelman_rubin([posterior_rmh, posterior_rmh_gt], n_most_frequent=None, plot=True, plot_show=False, file_name=posterior_rmh_gr_file_name)

    posterior_rmh_log_prob_file_name = os.path.join(results_dir, 'posterior_rmh_log_prob')
    pyprob.diagnostics.log_prob([posterior_rmh, posterior_rmh_gt], plot=True, plot_show=False, file_name=posterior_rmh_log_prob_file_name)

    pyprob.util.create_path(os.path.join(results_dir, 'addresses'), directory=True)
    pyprob.util.create_path(os.path.join(results_dir, 'addresses_aggregated'), directory=True)
    pyprob.util.create_path(os.path.join(results_dir, 'graph'), directory=True)
    pyprob.util.create_path(os.path.join(results_dir, 'graph_aggregated'), directory=True)

    posterior_is_rmh_addresses_file_name = os.path.join(results_dir, 'addresses/posterior_is_rmh_addresses')
    pyprob.diagnostics.address_histograms([posterior_rmh, proposal_is, posterior_is], plot=True, plot_show=False, ground_truth_trace=ground_truth_trace, use_address_base=False, file_name=posterior_is_rmh_addresses_file_name)

    posterior_is_rmh_addresses_aggregated_file_name = os.path.join(results_dir, 'addresses_aggregated/posterior_is_rmh_addresses_aggregated')
    pyprob.diagnostics.address_histograms([posterior_rmh, proposal_is, posterior_is], plot=True, plot_show=False, ground_truth_trace=ground_truth_trace, use_address_base=True, file_name=posterior_is_rmh_addresses_aggregated_file_name)

    posterior_ic_rmh_addresses_file_name = os.path.join(results_dir, 'addresses/posterior_ic_rmh_addresses')
    pyprob.diagnostics.address_histograms([posterior_rmh, proposal_ic, posterior_ic], plot=True, plot_show=False, ground_truth_trace=ground_truth_trace, use_address_base=False, file_name=posterior_ic_rmh_addresses_file_name)

    posterior_ic_rmh_addresses_aggregated_file_name = os.path.join(results_dir, 'addresses_aggregated/posterior_ic_rmh_addresses_aggregated')
    pyprob.diagnostics.address_histograms([posterior_rmh, proposal_ic, posterior_ic], plot=True, plot_show=False, ground_truth_trace=ground_truth_trace, use_address_base=True, file_name=posterior_ic_rmh_addresses_aggregated_file_name)

    posterior_is_graph_file_name = os.path.join(results_dir, 'graph/posterior_is_graph')
    pyprob.diagnostics.graph(posterior_is, file_name=posterior_is_graph_file_name)

    posterior_is_graph_aggregated_file_name = os.path.join(results_dir, 'graph_aggregated/posterior_is_graph_aggregated')
    pyprob.diagnostics.graph(posterior_is, use_address_base=True, file_name=posterior_is_graph_aggregated_file_name)

    posterior_ic_graph_file_name = os.path.join(results_dir, 'graph/posterior_ic_graph')
    pyprob.diagnostics.graph(posterior_ic, file_name=posterior_ic_graph_file_name)

    posterior_ic_graph_aggregated_file_name = os.path.join(results_dir, 'graph_aggregated/posterior_ic_graph_aggregated')
    pyprob.diagnostics.graph(posterior_ic, use_address_base=True, file_name=posterior_ic_graph_aggregated_file_name)

    posterior_rmh_graph_file_name = os.path.join(results_dir, 'graph/posterior_rmh_graph')
    pyprob.diagnostics.graph(posterior_rmh, file_name=posterior_rmh_graph_file_name)

    posterior_rmh_graph_aggregated_file_name = os.path.join(results_dir, 'graph_aggregated/posterior_rmh_graph_aggregated')
    pyprob.diagnostics.graph(posterior_rmh, use_address_base=True, file_name=posterior_rmh_graph_aggregated_file_name)

    posterior_is.close()
    posterior_ic.close()
    posterior_rmh.close()
    posterior_rmh_gt.close()

if __name__ == '__main__':
    pyprob.seed(1)

    current_dir = os.path.dirname(os.path.abspath(__file__))
    print('Current dir: {}'.format(current_dir))

    results_dir = os.path.join(current_dir, 'gum_marsaglia/replace_true')
    produce_results(replace=True, results_dir=results_dir)

    results_dir = os.path.join(current_dir, 'gum_marsaglia/replace_false')
    produce_results(replace=False, results_dir=results_dir)

    print('Done')


# Modified by BaseDetection, Inc. and its affiliates.

import contextlib
import copy
import datetime
import io
import json
import logging
import os
import os.path as osp

import numpy as np
from PIL import Image

import torch

from cvpods.structures import Boxes, BoxMode, PolygonMasks
from cvpods.utils import PathManager, Timer, file_lock

from ..base_dataset import BaseDataset
from ..detection_utils import (
    annotations_to_instances,
    check_image_size,
    create_keypoint_hflip_indices,
    filter_empty_instances,
    read_image
)
from ..registry import DATASETS
from .builtin_meta import _get_builtin_metadata
from .paths_route import _PREDEFINED_SPLITS_COCO

"""
This file contains functions to parse COCO-format annotations into dicts in "cvpods format".
"""

logger = logging.getLogger(__name__)

@DATASETS.register()
class COCODataset(BaseDataset):
    def __init__(self, cfg, dataset_name, transforms=[], is_train=True):
        super(COCODataset, self).__init__(cfg, dataset_name, transforms, is_train)

        if "panoptic" in dataset_name:
            self.task_key = "panoptic"      # for task: panoptic/semantic segmentation
        elif "keypoints" in dataset_name:
            self.task_key = "coco_person"   # for task: keypoints detection
        else:
            self.task_key = "coco"          # for task: instance detection/segmentation

        self.meta = self._get_metadata()

        if self.task_key in ["coco", "coco_person"]:
            self.dataset_dicts = self._load_annotations(
                self.meta["json_file"],
                self.meta["image_root"],
                dataset_name)
        elif self.task_key in ["panoptic"]:
            # panoptic segmentation task, support below dataset names:
            if "_separated" in dataset_name:
                self.dataset_dicts = self._load_annotations(
                    self.meta["json_file"],
                    self.meta["image_root"],
                    dataset_name
                )
                dicts4seg = load_sem_seg(
                    self.meta["sem_seg_root"],
                    self.meta["image_root"],
                )

                assert len(self.dataset_dicts) == len(dicts4seg), \
                    "len(self.dataset_dicts): {}, len(dicts4seg): {}".format(
                        len(self.dataset_dicts), len(dicts4seg))

                for idx, (dataset_dict, dict4seg) in enumerate(zip(self.dataset_dicts, dicts4seg)):
                    assert dataset_dict['file_name'] == dict4seg['file_name'], \
                        "idx: {}, dataset_dict['file_name']: {}, dict4seg['file_name']: {}".format(
                            idx, dataset_dict['file_name'], dict4seg['file_name'])

                    assert "sem_seg_file_name" not in dataset_dict

                    dataset_dict["sem_seg_file_name"] = dict4seg["sem_seg_file_name"]

            # semantic segmentation task, support below dataset names:
            elif "_stuffonly" in dataset_name:
                self.dataset_dicts = load_sem_seg(
                    self.meta["sem_seg_root"],
                    self.meta["image_root"],
                )
            else:
                raise ValueError(f"Unknow dataset name: {self.name}.")

        # fmt: off
        self.data_format = cfg.INPUT.FORMAT
        self.mask_on = cfg.MODEL.MASK_ON
        self.mask_format = cfg.INPUT.MASK_FORMAT
        self.filter_empty = cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS
        self.keypoint_on = cfg.MODEL.KEYPOINT_ON
        self.load_proposals = cfg.MODEL.LOAD_PROPOSALS
        self.proposal_files = cfg.DATASETS.PROPOSAL_FILES_TRAIN
        # fmt: on

        if is_train:
            # Remove images without instance-level GT even though the dataset has semantic labels.
            self.dataset_dicts = self._filter_annotations(
                filter_empty=self.filter_empty,
                min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE
                if self.keypoint_on else 0,
                proposal_files=self.proposal_files if self.load_proposals else None,
            )
            self._set_group_flag()

        self.eval_with_gt = cfg.TEST.get("WITH_GT", False)

        if self.keypoint_on:
            # Flip only makes sense in training
            self.keypoint_hflip_indices = create_keypoint_hflip_indices(
                cfg.DATASETS.TRAIN, self.meta)
        else:
            self.keypoint_hflip_indices = None

    def __getitem__(self, index):
        """Load data, apply transforms, converto to Instances.
        """
        dataset_dict = copy.deepcopy(self.dataset_dicts[index])

        # read image
        image = read_image(dataset_dict["file_name"], format=self.data_format)
        check_image_size(dataset_dict, image)

        if "annotations" in dataset_dict:
            annotations = dataset_dict.pop("annotations")
            annotations = [
                ann for ann in annotations if ann.get("iscrowd", 0) == 0]
        else:
            annotations = None

        if "sem_seg_file_name" in dataset_dict:
            if annotations is None:
                annotations = []
            with PathManager.open(dataset_dict.get("sem_seg_file_name"), "rb") as f:
                sem_seg_gt = Image.open(f)
                sem_seg_gt = np.asarray(sem_seg_gt, dtype="uint8")

            annotations.insert(0, {"sem_seg": sem_seg_gt})

        # apply transfrom
        image, annotations = self._apply_transforms(
            image, annotations, keypoint_hflip_indices=self.keypoint_hflip_indices)

        if "sem_seg_file_name" in dataset_dict:
            dataset_dict.pop("sem_seg_file_name")
            sem_seg_gt = annotations[0].pop("sem_seg")
            sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long"))
            dataset_dict["sem_seg"] = sem_seg_gt
            annotations = annotations[1:]

            if not annotations:
                annotations = None

        if annotations is not None:  # got instances in annotations
            image_shape = image.shape[:2]  # h, w

            instances = annotations_to_instances(
                annotations, image_shape, mask_format=self.mask_format
            )

            # # Create a tight bounding box from masks, useful when image is cropped

            dataset_dict["instances"] = filter_empty_instances(instances)

        # convert to Instance type
        # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory,
        # but not efficient on large generic data structures due to the use of pickle & mp.Queue.
        # Therefore it's important to use torch.Tensor.
        dataset_dict["image"] = torch.as_tensor(
            np.ascontiguousarray(image.transpose(2, 0, 1)))

        return dataset_dict

    def __reset__(self):
        raise NotImplementedError

    def __len__(self):
        return len(self.dataset_dicts)

    def _load_annotations(self,
                          json_file,
                          image_root,
                          dataset_name=None,
                          extra_annotation_keys=None):
        """
        Load a json file with COCO's instances annotation format.
        Currently supports instance detection, instance segmentation,
        and person keypoints annotations.

        Args:
            json_file (str): full path to the json file in COCO instances annotation format.
            image_root (str): the directory where the images in this json file exists.
            dataset_name (str): the name of the dataset (e.g., coco_2017_train).
                If provided, this function will also put "thing_classes" into
                the metadata associated with this dataset.
            extra_annotation_keys (list[str]): list of per-annotation keys that should also be
                loaded into the dataset dict (besides "iscrowd", "bbox", "keypoints",
                "category_id", "segmentation"). The values for these keys will be returned as-is.
                For example, the densepose annotations are loaded in this way.

        Returns:
            list[dict]: a list of dicts in cvpods standard format. (See
            `Using Custom Datasets </tutorials/datasets.html>`_ )

        Notes:
            1. This function does not read the image files.
            The results do not have the "image" field.
        """
        from pycocotools.coco import COCO

        timer = Timer()
        json_file = PathManager.get_local_path(json_file)
        with contextlib.redirect_stdout(io.StringIO()):
            coco_api = COCO(json_file)
        if timer.seconds() > 1:
            logger.info("Loading {} takes {:.2f} seconds.".format(
                json_file, timer.seconds()))

        id_map = None
        if dataset_name is not None:
            cat_ids = sorted(coco_api.getCatIds())
            cats = coco_api.loadCats(cat_ids)
            # The categories in a custom json file may not be sorted.
            thing_classes = [
                c["name"] for c in sorted(cats, key=lambda x: x["id"])
            ]
            self.meta["thing_classes"] = thing_classes

            # In COCO, certain category ids are artificially removed,
            # and by convention they are always ignored.
            # We deal with COCO's id issue and translate
            # the category ids to contiguous ids in [0, 80).

            # It works by looking at the "categories" field in the json, therefore
            # if users' own json also have incontiguous ids, we'll
            # apply this mapping as well but print a warning.
            if not (min(cat_ids) == 1 and max(cat_ids) == len(cat_ids)):
                if "coco" not in dataset_name:
                    logger.warning("""
    Category ids in annotations are not in [1, #categories]! We'll apply a mapping for you.
    """)
            id_map = {v: i for i, v in enumerate(cat_ids)}
            self.meta["thing_dataset_id_to_contiguous_id"] = id_map

        # sort indices for reproducible results
        img_ids = sorted(coco_api.imgs.keys())
        # imgs is a list of dicts, each looks something like:
        imgs = coco_api.loadImgs(img_ids)
        # anns is a list[list[dict]], where each dict is an annotation
        # record for an object. The inner list enumerates the objects in an image
        # and the outer list enumerates over images. Example of anns[0]:
        anns = [coco_api.imgToAnns[img_id] for img_id in img_ids]

        if "minival" not in json_file:
            # The popular valminusminival & minival annotations for COCO2014 contain this bug.
            # However the ratio of buggy annotations there is tiny and does not affect accuracy.
            # Therefore we explicitly white-list them.
            ann_ids = [
                ann["id"] for anns_per_image in anns for ann in anns_per_image
            ]
            assert len(set(ann_ids)) == len(
                ann_ids), "Annotation ids in '{}' are not unique!".format(
                    json_file)

        imgs_anns = list(zip(imgs, anns))

        logger.info("Loaded {} images in COCO format from {}".format(
            len(imgs_anns), json_file))

        dataset_dicts = []

        ann_keys = ["iscrowd", "bbox", "keypoints", "category_id"
                    ] + (extra_annotation_keys or [])

        num_instances_without_valid_segmentation = 0

        for (img_dict, anno_dict_list) in imgs_anns:
            record = {}
            record["file_name"] = os.path.join(image_root,
                                               img_dict["file_name"])
            record["height"] = img_dict["height"]
            record["width"] = img_dict["width"]
            image_id = record["image_id"] = img_dict["id"]

            objs = []
            for anno in anno_dict_list:
                # Check that the image_id in this annotation is the same as
                # the image_id we're looking at.
                # This fails only when the data parsing logic or the annotation file is buggy.

                # The original COCO valminusminival2014 & minival2014 annotation files
                # actually contains bugs that, together with certain ways of using COCO API,
                # can trigger this assertion.
                assert anno["image_id"] == image_id
                assert anno.get("ignore", 0) == 0

                obj = {key: anno[key] for key in ann_keys if key in anno}

                segm = anno.get("segmentation", None)
                if segm:    # either list[list[float]] or dict(RLE)
                    if not isinstance(segm, dict):
                        segm = [
                            poly for poly in segm
                            if len(poly) % 2 == 0 and len(poly) >= 6
                        ]
                        if len(segm) == 0:
                            num_instances_without_valid_segmentation += 1
                            continue    # ignore this instance
                    obj["segmentation"] = segm

                keypts = anno.get("keypoints", None)
                if keypts:    # list[int]
                    for idx, v in enumerate(keypts):
                        if idx % 3 != 2:
                            # COCO's segmentation coordinates are floating points in [0, H or W],
                            # but keypoint coordinates are integers in [0, H-1 or W-1]
                            # Therefore we assume the coordinates are "pixel indices" and
                            # add 0.5 to convert to floating point coordinates.
                            keypts[idx] = v + 0.5
                    obj["keypoints"] = keypts

                obj["bbox_mode"] = BoxMode.XYWH_ABS
                if id_map:
                    obj["category_id"] = id_map[obj["category_id"]]
                objs.append(obj)
            record["annotations"] = objs
            dataset_dicts.append(record)

        if num_instances_without_valid_segmentation > 0:
            logger.warning(
                "Filtered out {} instances without valid segmentation. "
                "There might be issues in your dataset generation process.".
                format(num_instances_without_valid_segmentation))
        return dataset_dicts

    def _get_metadata(self):
        if self.task_key in ["coco", "coco_person"]:
            meta = _get_builtin_metadata(self.task_key)
            image_root, json_file = _PREDEFINED_SPLITS_COCO[self.task_key][self.name]
            meta["image_root"] = osp.join(self.data_root, image_root) \
                if "://" not in image_root else image_root
            meta["json_file"] = osp.join(self.data_root, json_file) \
                if "://" not in image_root else osp.join(image_root, json_file)
            meta["evaluator_type"] = _PREDEFINED_SPLITS_COCO["evaluator_type"][self.task_key]
        elif self.task_key in ["panoptic"]:
            meta = _get_builtin_metadata("coco_panoptic_separated")
            prefix_instances = self.name[: -len("_panoptic_separated")]
            prefix_panoptic = self.name[: -len("_separated")]
            eval_key = self.name[-len("panoptic_separated"):]

            image_root, json_file = _PREDEFINED_SPLITS_COCO["coco"][prefix_instances]
            panoptic_root, panoptic_json, semantic_root = \
                _PREDEFINED_SPLITS_COCO[self.task_key][prefix_panoptic]
            meta["image_root"] = osp.join(self.data_root, image_root) \
                if "://" not in image_root else image_root
            meta["sem_seg_root"] = os.path.join(self.data_root, semantic_root) \
                if "://" not in semantic_root else semantic_root
            meta["evaluator_type"] = _PREDEFINED_SPLITS_COCO["evaluator_type"][eval_key]

            if "_separated" in self.name:
                meta["json_file"] = osp.join(self.data_root, json_file) \
                    if "://" not in image_root else osp.join(image_root, json_file)
                meta["panoptic_root"] = os.path.join(self.data_root, panoptic_root) \
                    if "://" not in panoptic_root else panoptic_root
                meta["panoptic_json"] = os.path.join(self.data_root, panoptic_json) \
                    if "://" not in panoptic_root else osp.join(panoptic_root, panoptic_json)
        return meta

    def evaluate(self, predictions):
        """Dataset must provide a evaluation function to evaluate model."""

    @property
    def ground_truth_annotations(self):
        return self.dataset_dicts

# TODO this function is not specific to COCO, except for the "image_id" logic.
def load_sem_seg(gt_root, image_root, gt_ext="png", image_ext="jpg"):
    """
    Load semantic segmentation datasets. All files under "gt_root" with "gt_ext" extension are
    treated as ground truth annotations and all files under "image_root" with "image_ext" extension
    as input images. Ground truth and input images are matched using file paths relative to
    "gt_root" and "image_root" respectively without taking into account file extensions.
    This works for COCO as well as some other datasets.

    Args:
        gt_root (str): full path to ground truth semantic segmentation files. Semantic segmentation
            annotations are stored as images with integer values in pixels that represent
            corresponding semantic labels.
        image_root (str): the directory where the input images are.
        gt_ext (str): file extension for ground truth annotations.
        image_ext (str): file extension for input images.

    Returns:
        list[dict]:
            a list of dicts in cvpods standard format without instance-level
            annotation.

    Notes:
        1. This function does not read the image and ground truth files.
           The results do not have the "image" and "sem_seg" fields.
    """

    # We match input images with ground truth based on their relative filepaths (without file
    def file2id(folder_path, file_path):
        # extract relative path starting from `folder_path`
        image_id = os.path.normpath(
            os.path.relpath(file_path, start=folder_path))
        # remove file extension
        image_id = os.path.splitext(image_id)[0]
        return image_id

    input_files = sorted(
        (os.path.join(image_root, f)
         for f in PathManager.ls(image_root) if f.endswith(image_ext)),
        key=lambda file_path: file2id(image_root, file_path),
    )
    gt_files = sorted(
        (os.path.join(gt_root, f)
         for f in PathManager.ls(gt_root) if f.endswith(gt_ext)),
        key=lambda file_path: file2id(gt_root, file_path),
    )

    assert len(gt_files) > 0, "No annotations found in {}.".format(gt_root)

    # Use the intersection, so that val2017_100 annotations can run smoothly with val2017 images
    if len(input_files) != len(gt_files):
        logger.warn(
            "Directory {} and {} has {} and {} files, respectively.".format(
                image_root, gt_root, len(input_files), len(gt_files)))
        input_basenames = [
            os.path.basename(f)[:-len(image_ext)] for f in input_files
        ]
        gt_basenames = [os.path.basename(f)[:-len(gt_ext)] for f in gt_files]
        intersect = list(set(input_basenames) & set(gt_basenames))
        # sort, otherwise each worker may obtain a list[dict] in different order
        intersect = sorted(intersect)
        logger.warn("Will use their intersection of {} files.".format(
            len(intersect)))
        input_files = [
            os.path.join(image_root, f + image_ext) for f in intersect
        ]
        gt_files = [os.path.join(gt_root, f + gt_ext) for f in intersect]

    logger.info("Loaded {} images with semantic segmentation from {}".format(
        len(input_files), image_root))

    dataset_dicts = []
    for (img_path, gt_path) in zip(input_files, gt_files):
        record = {}
        record["file_name"] = img_path
        record["sem_seg_file_name"] = gt_path
        dataset_dicts.append(record)

    return dataset_dicts

def convert_to_coco_dict(dataset_name, dataset_dicts, metadata):
    """
    Convert a dataset in cvpods's standard format into COCO json format
    COCO data format description can be found here:
    http://example.com
    Args:
        dataset_name:
            name of the source dataset
            must be registered in DatastCatalog and in cvpods's standard format
    Returns:
        coco_dict: serializable dict in COCO json format
    """
    if dataset_name not in [
        "citypersons_train",
        "citypersons_val",
        "crowdhuman_train",
        "crowdhuman_val",
        "coco_2017_train",
        "coco_2017_val",
        "widerface_2019_train",
        "widerface_2019_val"
    ]:
        raise NotImplementedError("Dataset name '{}' not supported".format(dataset_name))

    # unmap the category mapping ids for COCO
    if hasattr(metadata, "thing_dataset_id_to_contiguous_id"):
        reverse_id_mapping = {
            v: k
            for k, v in metadata.thing_dataset_id_to_contiguous_id.items()
        }

        def reverse_id_mapper(contiguous_id): return reverse_id_mapping[contiguous_id]  # noqa
    else:
        def reverse_id_mapper(contiguous_id): return contiguous_id    # noqa

    categories = [{
        "id": reverse_id_mapper(id),
        "name": name
    } for id, name in enumerate(metadata.thing_classes)]

    logger.info("Converting dataset dicts into COCO format")
    coco_images = []
    coco_annotations = []

    for image_id, image_dict in enumerate(dataset_dicts):
        coco_image = {
            "id": image_dict.get("image_id", image_id),
            "width": image_dict["width"],
            "height": image_dict["height"],
            "file_name": image_dict["file_name"],
        }
        coco_images.append(coco_image)

        anns_per_image = image_dict["annotations"]
        for annotation in anns_per_image:
            # create a new dict with only COCO fields
            coco_annotation = {}

            # COCO requirement: XYWH box format
            bbox = annotation["bbox"]
            bbox_mode = annotation["bbox_mode"]
            bbox = BoxMode.convert(bbox, bbox_mode, BoxMode.XYWH_ABS)

            # COCO requirement: instance area
            if "segmentation" in annotation:
                # Computing areas for instances by counting the pixels
                segmentation = annotation["segmentation"]
                # TODO: check segmentation type: RLE, BinaryMask or Polygon
                polygons = PolygonMasks([segmentation])
                area = polygons.area()[0].item()
            else:
                # Computing areas using bounding boxes
                bbox_xy = BoxMode.convert(bbox, BoxMode.XYWH_ABS,
                                          BoxMode.XYXY_ABS)
                area = Boxes([bbox_xy]).area()[0].item()

            if "keypoints" in annotation:
                keypoints = annotation["keypoints"]    # list[int]
                for idx, v in enumerate(keypoints):
                    if idx % 3 != 2:
                        # COCO's segmentation coordinates are floating points in [0, H or W],
                        # but keypoint coordinates are integers in [0, H-1 or W-1]
                        # For COCO format consistency we substract 0.5
                        keypoints[idx] = v - 0.5
                if "num_keypoints" in annotation:
                    num_keypoints = annotation["num_keypoints"]
                else:
                    num_keypoints = sum(kp > 0 for kp in keypoints[2::3])

            # COCO requirement:
            #   linking annotations to images
            #   "id" field must start with 1
            coco_annotation["id"] = len(coco_annotations) + 1
            coco_annotation["image_id"] = coco_image["id"]
            coco_annotation["bbox"] = [round(float(x), 3) for x in bbox]
            coco_annotation["area"] = area
            coco_annotation["category_id"] = reverse_id_mapper(
                annotation["category_id"])
            coco_annotation["iscrowd"] = annotation.get("iscrowd", 0)

            # Add optional fields
            if "keypoints" in annotation:
                coco_annotation["keypoints"] = keypoints
                coco_annotation["num_keypoints"] = num_keypoints

            if "segmentation" in annotation:
                coco_annotation["segmentation"] = annotation["segmentation"]

            coco_annotations.append(coco_annotation)

    logger.info(
        "Conversion finished, "
        f"num images: {len(coco_images)}, num annotations: {len(coco_annotations)}"
    )

    info = {
        "date_created": str(datetime.datetime.now()),
        "description":
        "Automatically generated COCO json file for cvpods.",
    }
    coco_dict = {
        "info": info,
        "images": coco_images,
        "annotations": coco_annotations,
        "categories": categories,
        "licenses": None,
    }
    return coco_dict

def convert_to_coco_json(dataset_name, output_file, allow_cached=True):
    """
    Converts dataset into COCO format and saves it to a json file.
    dataset_name must be registered in DatasetCatalog and in cvpods's standard format.
    Args:
        dataset_name:
            reference from the config file to the catalogs
            must be registered in DatasetCatalog and in cvpods's standard format
        output_file: path of json file that will be saved to
        allow_cached: if json file is already present then skip conversion
    """

    # TODO: The dataset or the conversion script *may* change,
    # a checksum would be useful for validating the cached data

    PathManager.mkdirs(os.path.dirname(output_file))
    with file_lock(output_file):
        if PathManager.exists(output_file) and allow_cached:
            logger.info(
                f"Cached annotations in COCO format already exist: {output_file}"
            )
        else:
            logger.info(
                f"Converting dataset annotations in '{dataset_name}' to COCO format ...)"
            )
            coco_dict = convert_to_coco_dict(dataset_name)

            with PathManager.open(output_file, "w") as json_file:
                logger.info(
                    f"Caching annotations in COCO format: {output_file}")
                json.dump(coco_dict, json_file)


    key_parts = []
    path_parts = path.strip(u'/').split(u'/')
    for n, x in enumerate(path_parts):
        name, ext = x.rsplit('.', 1)
        key_parts.extend([ext, name])

    return datastore.key(*key_parts)

import argparse
import datetime

from google.cloud import datastore

def path_to_key(datastore, path):
    """
    Translates a file system path to a datastore key. The basename becomes the
    key name and the extension becomes the kind.

    Examples:
        /file.ext -> key(ext, file)
        /parent.ext/file.ext -> key(ext, parent, ext, file)
    """

def create_user(ds, username, profile):
    key = path_to_key(ds, '{0}.user'.format(username))
    entity = datastore.Entity(key)
    entity.update(profile)
    ds.put(entity)

def create_post(ds, username, post_content):
    now = datetime.datetime.utcnow()
    key = path_to_key(ds, '{0}.user/{1}.post'.format(username, now))
    entity = datastore.Entity(key)

    entity.update({
        'created': now,
        'created_by': username,
        'content': post_content
    })

    ds.put(entity)

def repost(ds, username, original):
    now = datetime.datetime.utcnow()
    new_key = path_to_key(ds, '{0}.user/{1}.post'.format(username, now))
    new = datastore.Entity(new_key)

    new.update(original)

    ds.put(new)

def list_posts_by_user(ds, username):
    user_key = path_to_key(ds, '{0}.user'.format(username))
    return ds.query(kind='post', ancestor=user_key).fetch()

def list_all_posts(ds):
    return ds.query(kind='post').fetch()

def main(project_id):
    ds = datastore.Client(project_id)

    print("Creating users...")
    create_user(ds, 'tonystark',
                {'name': 'Tony Stark', 'location': 'Stark Island'})
    create_user(ds, 'peterparker',
                {'name': 'Peter Parker', 'location': 'New York City'})

    print("Creating posts...")
    for n in range(1, 10):
        create_post(ds, 'tonystark', "Tony's post #{0}".format(n))
        create_post(ds, 'peterparker', "Peter's post #{0}".format(n))

    print("Re-posting tony's post as peter...")

    tonysposts = list_posts_by_user(ds, 'tonystark')
    for post in tonysposts:
        original_post = post
        break

    repost(ds, 'peterparker', original_post)

    print('Posts by tonystark:')
    for post in list_posts_by_user(ds, 'tonystark'):
        print("> {0} on {1}".format(post['content'], post['created']))

    print('Posts by peterparker:')
    for post in list_posts_by_user(ds, 'peterparker'):
        print("> {0} on {1}".format(post['content'], post['created']))

    print('Posts by everyone:')
    for post in list_all_posts(ds):
        print("> {0} on {1}".format(post['content'], post['created']))

    print('Cleaning up...')
    ds.delete_multi([
        path_to_key(ds, 'tonystark.user'),
        path_to_key(ds, 'peterparker.user')
    ])
    ds.delete_multi([
        x.key for x in list_all_posts(ds)])

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='Demonstrates wiki data model.')
    parser.add_argument('project_id', help='Your cloud project ID.')

    args = parser.parse_args()

    main(args.project_id)


while idx >= 0 and view.substr(idx) == '\\':
            escaped = ~escaped
            idx -= 1

import bh_plugin
import sublime

class SwapQuotes(bh_plugin.BracketPluginCommand):
    def escaped(self, idx):
        view = self.view
        escaped = False

        return escaped

    def run(self, edit, name):
        view = self.view
        quote = view.substr(self.left.begin)
        if quote != "'" and quote != '"':
            return
        new = "'" if (quote == '"') else '"'
        old = quote
        begin = self.left.end
        end = self.right.begin
        content_end = self.right.begin

        view.replace(edit, self.left.toregion(), view.substr(self.left.toregion()).replace(old, new))
        view.replace(edit, self.right.toregion(), view.substr(self.right.toregion()).replace(old, new))

        offset = 0
        while begin < end + offset:
            char = view.substr(begin)
            if char == old and self.escaped(begin - 1):
                view.replace(edit, sublime.Region(begin - 1, begin), '')
                offset -= 1
                content_end -= 1
            elif char == new and not self.escaped(begin - 1):
                view.insert(edit, begin, "\\")
                offset += 1
                content_end += 1
            begin += 1

        self.right = self.right.move(content_end, end + offset)
        self.selection = [sublime.Region(content_end)]

def plugin():
    return SwapQuotes