# -*- coding: utf-8 -*- #
# Copyright 2013 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Manages the state of what is installed in the cloud SDK.

This tracks the installed modules along with the files they created.  It also
provides functionality like extracting tar files into the installation and
tracking when we check for updates.
"""


import compileall
import errno
import logging
import os
import posixpath
import re
import shutil
import sys

from googlecloudsdk.core import config
from googlecloudsdk.core import exceptions
from googlecloudsdk.core.console import console_io
from googlecloudsdk.core.updater import installers
from googlecloudsdk.core.updater import snapshots
from googlecloudsdk.core.util import encoding
from googlecloudsdk.core.util import files as file_utils

import six


class Error(exceptions.Error):
  """Base exception for the local_state module."""
  pass


class InvalidSDKRootError(Error):
  """Error for when the root of the Cloud SDK is invalid or cannot be found."""

  def __init__(self):
    super(InvalidSDKRootError, self).__init__(
        'You cannot perform this action because this Cloud SDK '
        'installation is managed by an external package manager.\n'
        'Please consider using a separate installation of the Cloud '
        'SDK created through the default mechanism described at: '
        '{doc_url}\n'.format(
            doc_url=config.INSTALLATION_CONFIG.documentation_url))



class InvalidDownloadError(Error):
  """Exception for when the SDK that was download was invalid."""

  def __init__(self):
    super(InvalidDownloadError, self).__init__(
        'The Cloud SDK download was invalid.')


class PermissionsError(Error):
  """Error for when a file operation cannot complete due to permissions."""

  def __init__(self, message, path):
    """Initialize a PermissionsError.

    Args:
      message: str, The message from the underlying error.
      path: str, The absolute path to a file or directory that needs to be
          operated on, but can't because of insufficient permissions.
    """
    super(PermissionsError, self).__init__(
        '{message}: [{path}]\n\nEnsure you have the permissions to access the '
        'file and that the file is not in use.'
        .format(message=message, path=path))


def _RaisesPermissionsError(func):
  """Use this decorator for functions that deal with files.

  If an exception indicating file permissions is raised, this decorator will
  raise a PermissionsError instead, so that the caller only has to watch for
  one type of exception.

  Args:
    func: The function to decorate.

  Returns:
    A decorator.
  """

  def _TryFunc(*args, **kwargs):
    try:
      return func(*args, **kwargs)
    except shutil.Error as e:
      args = e.args[0][0]
      # unfortunately shutil.Error *only* has formatted strings to inspect.
      # Looking for this substring is looking for errno.EACCES, which has
      # a numeric value of 13.
      if args[2].startswith('[Errno 13]'):
        exceptions.reraise(
            PermissionsError(message=args[2],
                             path=os.path.abspath(args[0])))
      raise
    except (OSError, IOError) as e:
      if e.errno == errno.EACCES:
        exceptions.reraise(
            PermissionsError(
                message=encoding.Decode(e.strerror),
                path=encoding.Decode(os.path.abspath(e.filename))))
      raise
  return _TryFunc


class InstallationState(object):
  """The main class for checking / updating local installation state."""

  STATE_DIR_NAME = config.Paths.CLOUDSDK_STATE_DIR
  BACKUP_DIR_NAME = '.backup'
  TRASH_DIR_NAME = '.trash'
  STAGING_ROOT_SUFFIX = '.staging'
  COMPONENT_SNAPSHOT_FILE_SUFFIX = '.snapshot.json'
  DEPRECATED_DIRS = ('lib/third_party/grpc',)

  @staticmethod
  def ForCurrent():
    """Gets the installation state for the SDK that this code is running in.

    Returns:
      InstallationState, The state for this area.

    Raises:
      InvalidSDKRootError: If this code is not running under a valid SDK.
    """
    sdk_root = config.Paths().sdk_root
    if not sdk_root:
      raise InvalidSDKRootError()
    return InstallationState(os.path.realpath(sdk_root))

  def BackupInstallationState(self):
    """Gets the installation state for the backup of this  state, if it exists.

    Returns:
      InstallationState, The state for this area or None if the backup does not
          exist.
    """
    if not self.HasBackup():
      return None
    return InstallationState(os.path.realpath(self.__backup_directory))

  @staticmethod
  def VersionForInstalledComponent(component_id):
    """Gets the version string for the given installed component.

    This function is to be used to get component versions for metrics reporting.
    If it fails in any way or if the component_id is unknown, it will return
    None.  This prevents errors from surfacing when the version is needed
    strictly for reporting purposes.

    Args:
      component_id: str, The component id of the component you want the version
        for.

    Returns:
      str, The installed version of the component, or None if it is not
        installed or if an error occurs.
    """
    try:
      state = InstallationState.ForCurrent()
      # pylint: disable=protected-access, This is the same class.
      return InstallationManifest(
          state._state_directory, component_id).VersionString()
    # pylint: disable=bare-except, We never want to fail because of metrics.
    except:
      logging.debug('Failed to get installed version for component [%s]: [%s]',
                    component_id, sys.exc_info())
    return None

  @_RaisesPermissionsError
  def __init__(self, sdk_root):
    """Initializes the installation state for the given sdk install.

    Args:
      sdk_root: str, The file path of the root of the SDK installation.

    Raises:
      ValueError: If the given SDK root does not exist.
    """
    if not os.path.isdir(sdk_root):
      raise ValueError('The given Cloud SDK root does not exist: [{0}]'
                       .format(sdk_root))

    self.__sdk_root = encoding.Decode(sdk_root)
    self._state_directory = os.path.join(self.__sdk_root,
                                         InstallationState.STATE_DIR_NAME)
    self.__backup_directory = os.path.join(self._state_directory,
                                           InstallationState.BACKUP_DIR_NAME)
    self.__trash_directory = os.path.join(self._state_directory,
                                          InstallationState.TRASH_DIR_NAME)

    self.__sdk_staging_root = (os.path.normpath(self.__sdk_root) +
                               InstallationState.STAGING_ROOT_SUFFIX)
    self._component_installer = installers.ComponentInstaller(
        self.__sdk_root, self._state_directory)

  @_RaisesPermissionsError
  def _CreateStateDir(self):
    """Creates the state directory if it does not exist."""
    if not os.path.isdir(self._state_directory):
      file_utils.MakeDir(self._state_directory)

  @property
  def sdk_root(self):
    """Gets the root of the SDK that this state corresponds to.

    Returns:
      str, the path to the root directory.
    """
    return self.__sdk_root

  def _FilesForSuffix(self, suffix):
    """Returns the files in the state directory that have the given suffix.

    Args:
      suffix: str, The file suffix to match on.

    Returns:
      list of str, The file names that match.
    """
    if not os.path.isdir(self._state_directory):
      return []

    files = os.listdir(self._state_directory)
    matching = [f for f in files
                if os.path.isfile(os.path.join(self._state_directory, f))
                and f.endswith(suffix)]
    return matching

  @_RaisesPermissionsError
  def InstalledComponents(self):
    """Gets all the components that are currently installed.

    Returns:
      A dictionary of component id string to InstallationManifest.
    """
    snapshot_files = self._FilesForSuffix(
        InstallationState.COMPONENT_SNAPSHOT_FILE_SUFFIX)
    manifests = {}
    for f in snapshot_files:
      component_id = f[:-len(InstallationState.COMPONENT_SNAPSHOT_FILE_SUFFIX)]
      manifests[component_id] = InstallationManifest(self._state_directory,
                                                     component_id)
    return manifests

  @_RaisesPermissionsError
  def Snapshot(self):
    """Generates a ComponentSnapshot from the currently installed components."""
    return snapshots.ComponentSnapshot.FromInstallState(self)

  def DiffCurrentState(self, latest_snapshot, platform_filter=None,):
    """Generates a ComponentSnapshotDiff from current state and the given state.

    Args:
      latest_snapshot:  snapshots.ComponentSnapshot, The current state of the
        world to diff against.
      platform_filter: platforms.Platform, A platform that components must
        match in order to be considered for any operations.

    Returns:
      A ComponentSnapshotDiff.
    """
    return self.Snapshot().CreateDiff(latest_snapshot,
                                      platform_filter=platform_filter)

  @_RaisesPermissionsError
  def CreateStagingFromDownload(self, url, progress_callback=None):
    """Creates a new staging area from a fresh download of the Cloud SDK.

    Args:
      url: str, The url to download the new SDK from.
      progress_callback: f(float), A function to call with the fraction of
        completeness.

    Returns:
      An InstallationState object for the new install.

    Raises:
      installers.URLFetchError: If the new SDK could not be downloaded.
      InvalidDownloadError: If the new SDK was malformed.
    """
    self._ClearStaging()

    with file_utils.TemporaryDirectory() as t:
      download_dir = os.path.join(t, '.download')
      extract_dir = os.path.join(t, '.extract')
      (download_callback, extract_callback) = (
          console_io.SplitProgressBar(progress_callback, [1, 1]))
      downloaded_tar = installers.DownloadTar(
          url, download_dir, progress_callback=download_callback,
          command_path='components.reinstall')
      installers.ExtractTar(
          downloaded_tar, extract_dir, progress_callback=extract_callback)
      files = os.listdir(extract_dir)
      if len(files) != 1:
        raise InvalidDownloadError()
      sdk_root = os.path.join(extract_dir, files[0])
      file_utils.MoveDir(sdk_root, self.__sdk_staging_root)

    staging_sdk = InstallationState(self.__sdk_staging_root)
    # pylint: disable=protected-access, This is an instance of InstallationState
    staging_sdk._CreateStateDir()
    self.CopyMachinePropertiesTo(staging_sdk)
    return staging_sdk

  @_RaisesPermissionsError
  def ReplaceWith(self, other_install_state, progress_callback=None):
    """Replaces this installation with the given other installation.

    This moves the current installation to the backup directory of the other
    installation.  Then, it moves the entire second installation to replace
    this one on the file system.  The result is that the other installation
    completely replaces the current one, but the current one is snapshotted and
    stored as a backup under the new one (and can be restored later).

    Args:
      other_install_state: InstallationState, The other state with which to
        replace this one.
      progress_callback: f(float), A function to call with the fraction of
        completeness.
    """
    self._CreateStateDir()
    self.ClearBackup()
    self.ClearTrash()
    # pylint: disable=protected-access, This is an instance of InstallationState
    other_install_state._CreateStateDir()
    other_install_state.ClearBackup()
    # pylint: disable=protected-access, This is an instance of InstallationState
    file_utils.MoveDir(self.__sdk_root, other_install_state.__backup_directory)
    if progress_callback:
      progress_callback(0.5)
    file_utils.MoveDir(other_install_state.__sdk_root, self.__sdk_root)
    if progress_callback:
      progress_callback(1.0)

  @_RaisesPermissionsError
  def RestoreBackup(self):
    """Restore the backup from this install state if it exists.

    If this installation has a backup stored in it (created by and update that
    used ReplaceWith(), above), it replaces this installation with the backup,
    using a temporary staging area.  This installation is moved to the trash
    directory under the installation that exists after this is done.  The trash
    directory can be removed at any point in the future.  We just don't want to
    delete code that is running since some platforms have a problem with that.

    Returns:
      bool, True if there was a backup to restore, False otherwise.
    """
    if not self.HasBackup():
      return False

    self._ClearStaging()

    file_utils.MoveDir(self.__backup_directory, self.__sdk_staging_root)
    staging_state = InstallationState(self.__sdk_staging_root)
    # pylint: disable=protected-access, This is an instance of InstallationState
    staging_state._CreateStateDir()
    staging_state.ClearTrash()
    # pylint: disable=protected-access, This is an instance of InstallationState
    file_utils.MoveDir(self.__sdk_root, staging_state.__trash_directory)
    file_utils.MoveDir(staging_state.__sdk_root, self.__sdk_root)
    return True

  def HasBackup(self):
    """Determines if this install has a valid backup that can be restored.

    Returns:
      bool, True if there is a backup, False otherwise.
    """
    return os.path.isdir(self.__backup_directory)

  def BackupDirectory(self):
    """Gets the backup directory of this installation if it exists.

    Returns:
      str, The path to the backup directory or None if it does not exist.
    """
    if self.HasBackup():
      return self.__backup_directory
    return None

  @_RaisesPermissionsError
  def _ClearStaging(self, progress_callback=None):
    """Deletes the current staging directory if it exists.

    Args:
      progress_callback: f(float), A function to call with the fraction of
        completeness.
    """
    if os.path.exists(self.__sdk_staging_root):
      file_utils.RmTree(self.__sdk_staging_root)
    if progress_callback:
      progress_callback(1)

  @_RaisesPermissionsError
  def ClearBackup(self, progress_callback=None):
    """Deletes the current backup if it exists.

    Args:
      progress_callback: f(float), A function to call with the fraction of
        completeness.
    """
    if os.path.isdir(self.__backup_directory):
      file_utils.RmTree(self.__backup_directory)
    if progress_callback:
      progress_callback(1)

  @_RaisesPermissionsError
  def ClearTrash(self, progress_callback=None):
    """Deletes the current trash directory if it exists.

    Args:
      progress_callback: f(float), A function to call with the fraction of
        completeness.
    """
    if os.path.isdir(self.__trash_directory):
      file_utils.RmTree(self.__trash_directory)
    if progress_callback:
      progress_callback(1)

  @_RaisesPermissionsError
  def Download(self, snapshot, component_id, progress_callback=None,
               command_path='unknown'):
    """Downloads the given component based on the given snapshot.

    Args:
      snapshot: snapshots.ComponentSnapshot, The snapshot that describes the
        component to install.
      component_id: str, The component to install from the given snapshot.
      progress_callback: f(float), A function to call with the fraction of
        completeness.
      command_path: the command path to include in the User-Agent header if the
        URL is HTTP

    Returns:
      Optional[str], The path of the downloaded archive, or None if the
        component has no actual sources.

    Raises:
      installers.URLFetchError: If the component associated with the provided
        component ID has a URL that is not fetched correctly.
    """
    self._CreateStateDir()
    component = snapshot.ComponentFromId(component_id)
    downloaded_archive = self._component_installer.Download(
        component, progress_callback=progress_callback,
        command_path=command_path)
    return downloaded_archive

  @_RaisesPermissionsError
  def Install(self, snapshot, component_id, downloaded_archive,
              progress_callback=None):
    """Installs the archive previously downloaded from self.Download().

    Args:
      snapshot: snapshots.ComponentSnapshot, The snapshot that describes the
        component to install.
      component_id: str, The component to install from the given snapshot.
      downloaded_archive: Optional[str], The path to the archive downloaded
        previously.
      progress_callback: f(float), A function to call with the fraction of
        completeness.
    """
    files = self._component_installer.Extract(
        downloaded_archive, progress_callback=progress_callback)
    manifest = InstallationManifest(self._state_directory, component_id)
    manifest.MarkInstalled(snapshot, files)

  @_RaisesPermissionsError
  def Uninstall(self, component_id, progress_callback=None):
    """Uninstalls the given component.

    Deletes all the files for this component and marks it as no longer being
    installed.

    Args:
      component_id: str, The id of the component to uninstall.
      progress_callback: f(float), A function to call with the fraction of
        completeness.
    """
    manifest = InstallationManifest(self._state_directory, component_id)
    paths = manifest.InstalledPaths()
    total_paths = len(paths)
    root = self.__sdk_root

    dirs_to_remove = set()
    pycache_dirs = set()
    for num, p in enumerate(paths, start=1):
      path = os.path.join(root, p)
      if os.path.isfile(path) or os.path.islink(path):
        os.remove(path)
        dir_path = os.path.dirname(os.path.normpath(p))
        if p.endswith('.py'):
          # Python 2 processes leave behind .pyc files adjacent to the .py file;
          # clean these up for any .py files being removed.
          pyc_path = path + 'c'
          if os.path.isfile(pyc_path):
            os.remove(pyc_path)
          # Python 3 processes leave behind __pycache__ folders in the .py
          # file's directory; clean these up as well. Since the .pyc files
          # within have different suffixes depending on the Python version, and
          # the version of Python that compiled the file may differ from the
          # current one running, it's faster to just delete the whole folder
          # later instead of trying to match the file(s) here.
          pycache_dirs.add(os.path.join(root, dir_path, '__pycache__'))
        while dir_path:
          dirs_to_remove.add(os.path.join(root, dir_path))
          dir_path = os.path.dirname(dir_path)
      elif os.path.isdir(path):
        dirs_to_remove.add(os.path.normpath(path))
      if progress_callback:
        progress_callback(num / total_paths)

    for d in pycache_dirs:
      if os.path.isdir(d) and not os.path.islink(d):
        file_utils.RmTree(d)

    # Remove dirs from the bottom up.  Subdirs will always have a longer path
    # than it's parent.
    for d in sorted(dirs_to_remove, key=len, reverse=True):
      if os.path.isdir(d) and not os.path.islink(d) and not os.listdir(d):
        os.rmdir(d)

    manifest.MarkUninstalled()

  @_RaisesPermissionsError
  def ClearDeprecatedDirs(self):
    """Clear deprecated directories that were not removed correctly."""
    for d in self.DEPRECATED_DIRS:
      path = os.path.join(self.sdk_root, d)
      if os.path.isdir(path):
        file_utils.RmTree(path)

  def CopyMachinePropertiesTo(self, other_state):
    """Copy this state's properties file to another state.

    This is primarily intended to be used to maintain the machine properties
    file during a schema-change-induced reinstall.

    Args:
      other_state: InstallationState, The installation state of the fresh
          Cloud SDK that needs the properties file mirrored in.
    """
    my_properties = os.path.join(
        self.sdk_root, config.Paths.CLOUDSDK_PROPERTIES_NAME)
    other_properties = os.path.join(
        other_state.sdk_root, config.Paths.CLOUDSDK_PROPERTIES_NAME)
    if not os.path.exists(my_properties):
      return
    shutil.copyfile(my_properties, other_properties)

  def CompilePythonFiles(self, force=False, workers=None):
    """Attempts to compile all the python files into .pyc files.

    Args:
      force: boolean, passed to force option of compileall.compiledir,
      workers: int, can be used to explicitly set number of worker processes;
        otherwise we determine it automatically. Only set for testing.

    This does not raise exceptions if compiling a given file fails.
    """
    if (encoding.GetEncodedValue(
        os.environ, 'CLOUDSDK_SKIP_PY_COMPILATION') == '1'):
      logging.info('Skipping Python compilation as requested by environment.')
      return

    # Some python code shipped in the SDK is not 2 + 3 compatible.
    # Create execlusion patterns to avoid compilation errors.
    # This is pretty hacky, ideally we would have this information in the
    # component metadata and derive the exclusion patterns from that.
    # However, this is an ok short-term solution until we have bundled python.
    if six.PY2:
      regex_exclusion = re.compile('(httplib2/python3|typing/python3'
                                   '|platform/bq/third_party/yaml/lib3'
                                   '|third_party/google/api_core'
                                   '|third_party/google/auth'
                                   '|third_party/google/oauth2'
                                   '|third_party/overrides'
                                   '|third_party/proto'
                                   '|dulwich'
                                   '|gapic'
                                   '|pubsublite'
                                   '|pubsub/lite_subscriptions.py'
                                   '|logging_v2'
                                   '|platform/bundledpythonunix'
                                   '|pubsub_v1/services)')
    else:
      # Do not compile anything on python 3.4.x
      if sys.version_info[1] == 4:
        regex_exclusion = re.compile('.*')
      elif sys.version_info[1] >= 7:
        regex_exclusion = re.compile(
            '(kubernetes/utils/create_from_yaml.py'
            '|platform/google_appengine'
            '|gslib/vendored/boto/boto/iam/connection.py'
            '|gslib/vendored/boto/tests/'
            '|gslib/vendored/boto/boto/manage/volume.py'
            '|third_party/.*/python2/'
            '|third_party/yaml/[a-z]*.py'
            '|third_party/yaml/lib2/'
            '|third_party/antlr3/'
            '|appengine/'
            '|google/cloud/appengine_'
            '|google/cloud/bigquery_logging_v1'
            '|third_party/fancy_urllib/'
            '|third_party/pluggy/_callers.py'
            '|platform/bq/third_party/gflags'
            '|platform/ext-runtime/nodejs/test/'
            '|platform/gsutil/third_party/apitools/ez_setup'
            '|platform/gsutil/third_party/pyparsing'
            '|platform/gsutil/third_party/crcmod_osx/crcmod/test)')
      else:
        regex_exclusion = None

    # The self.sdk_root pathname could contain unicode chars and py_compile
    # chokes on unicode paths. Using relative paths from self.sdk_root works
    # around the problem.
    with file_utils.ChDir(self.sdk_root):
      to_compile = [
          os.path.join('bin', 'bootstrapping'),
          os.path.join('data', 'cli'),
          'lib',
          'platform',
      ]
      # There are diminishing returns to using more worker processes past a
      # certain point, so we cap it to a reasonable amount here.
      num_workers = min(os.cpu_count(), 8) if workers is None else workers
      for d in to_compile:
        # Using 2 for quiet, in python 2.7 this value is used as a bool in the
        # implementation and bool(2) is True. Starting in python 3.5 this
        # parameter was changed to a multilevel value, where 1 hides files
        # being processed and 2 suppresses output.
        compileall.compile_dir(
            d, rx=regex_exclusion, quiet=2, force=force, workers=num_workers)


class InstallationManifest(object):
  """Class to encapsulate the data stored in installation manifest files."""

  MANIFEST_SUFFIX = '.manifest'

  def __init__(self, state_dir, component_id):
    """Creates a new InstallationManifest.

    Args:
      state_dir: str, The directory path where install state is stored.
      component_id: str, The component id that you want to get the manifest for.
    """
    self.state_dir = state_dir
    self.id = component_id
    self.snapshot_file = os.path.join(
        self.state_dir,
        component_id + InstallationState.COMPONENT_SNAPSHOT_FILE_SUFFIX)
    self.manifest_file = os.path.join(
        self.state_dir,
        component_id + InstallationManifest.MANIFEST_SUFFIX)

  def MarkInstalled(self, snapshot, files):
    """Marks this component as installed with the given snapshot and files.

    This saves the ComponentSnapshot and writes the installed files to a
    manifest so they can be removed later.

    Args:
      snapshot: snapshots.ComponentSnapshot, The snapshot that was the source
        of the install.
      files: list of str, The files that were created by the installation.
    """
    with file_utils.FileWriter(self.manifest_file) as fp:
      for f in _NormalizeFileList(files):
        fp.write(f + '\n')
    snapshot.WriteToFile(self.snapshot_file, component_id=self.id)

  def MarkUninstalled(self):
    """Marks this component as no longer being installed.

    This does not actually uninstall the component, but rather just removes the
    snapshot and manifest.
    """
    for f in [self.manifest_file, self.snapshot_file]:
      if os.path.isfile(f):
        os.remove(f)

  def ComponentSnapshot(self):
    """Loads the local ComponentSnapshot for this component.

    Returns:
      The snapshots.ComponentSnapshot for this component.
    """
    return snapshots.ComponentSnapshot.FromFile(self.snapshot_file)

  def ComponentDefinition(self):
    """Loads the ComponentSnapshot and get the schemas.Component this component.

    Returns:
      The schemas.Component for this component.
    """
    return self.ComponentSnapshot().ComponentFromId(self.id)

  def VersionString(self):
    """Gets the version string of this component as it was installed.

    Returns:
      str, The installed version of this component.
    """
    return self.ComponentDefinition().version.version_string

  def InstalledPaths(self):
    """Gets the list of files and dirs created by installing this component.

    Returns:
      list of str, The files and directories installed by this component.
    """
    with file_utils.FileReader(self.manifest_file) as f:
      files = [line.rstrip() for line in f]
    return files


def _NormalizeFileList(file_list):
  """Removes non-empty directory entries and sorts resulting list."""
  parent_directories = set([])
  directories = set([])
  files = set([])
  for f in file_list:
    # Drops any trailing /.
    norm_file_path = posixpath.normpath(f)
    if f.endswith('/'):
      directories.add(norm_file_path + '/')
    else:
      files.add(norm_file_path)
    norm_file_path = os.path.dirname(norm_file_path)
    while norm_file_path:
      parent_directories.add(norm_file_path + '/')
      norm_file_path = os.path.dirname(norm_file_path)

  return sorted((directories - parent_directories) | files)
