ubuntu-touch-coreapps-reviewers team mailing list archive

[Alan Pope  <alan@xxxxxxxxxx>]

Alan Pope  has proposed merging lp:~popey/ubuntu-filemanager-app/add-click-deps into lp:ubuntu-filemanager-app.

Commit message:
Add basics to cmake file and necessary parts to pull in samba libs.

Requested reviews:
  Ubuntu File Manager Developers (ubuntu-filemanager-dev)

For more details, see:
https://code.launchpad.net/~popey/ubuntu-filemanager-app/add-click-deps/+merge/270287

First pass at adding get-click-deps from Stefano Verzegnassi to pull samba libs in during build process. Not final.
-- 
Your team Ubuntu File Manager Developers is requested to review the proposed merge of lp:~popey/ubuntu-filemanager-app/add-click-deps into lp:ubuntu-filemanager-app.

=== modified file 'CMakeLists.txt'
--- CMakeLists.txt	2014-09-08 10:45:41 +0000
+++ CMakeLists.txt	2015-09-07 10:44:39 +0000
@@ -61,6 +61,15 @@
     add_custom_target(com_ubuntu_calendar_CLICKFiles ALL SOURCES ${CLICK_FILES})
 
     install(FILES ${CMAKE_CURRENT_BINARY_DIR}/manifest.json filemanager.apparmor ${CONTENT_HUB_EXPORTER} DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+	MESSAGE("Grabbing upstream libs to ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs")
+    execute_process(
+        COMMAND mkdir ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs
+        COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/get-click-deps ${CMAKE_CURRENT_SOURCE_DIR}/filemanager-libs.json ${CLICK_ARCH} ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs
+    )
+	MESSAGE("Installing upstream libs from ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs/usr/lib/${ARCH_TRIPLET}/ to ${DATA_DIR}lib/${ARCH_TRIPLET}")
+	install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs/usr/lib/${ARCH_TRIPLET}/ DESTINATION ${DATA_DIR}lib/${ARCH_TRIPLET})
+
 else(CLICK_MODE)
     execute_process(
         COMMAND qmake -query QT_INSTALL_QML

=== added file 'filemanager-libs.json'
--- filemanager-libs.json	1970-01-01 00:00:00 +0000
+++ filemanager-libs.json	2015-09-07 10:44:39 +0000
@@ -0,0 +1,50 @@
+{
+    "armhf": [
+        {
+            "url": "http://ports.ubuntu.com/ubuntu-ports";,
+            "dist": "vivid",
+            "component": "main",
+            "packages": [
+                "libsmbclient",
+                "samba-libs",
+                "libtalloc2",
+                "libtevent0",
+                "libwbclient0",
+                "libldb1",
+                "libntdb1"
+            ]
+        }
+    ],
+    "amd64": [
+        {
+            "url": "http://archive.ubuntu.com/ubuntu";,
+            "dist": "vivid",
+            "component": "main",
+            "packages": [
+                "libsmbclient",
+                "samba-libs",
+                "libtalloc2",
+                "libtevent0",
+                "libwbclient0",
+                "libldb1",
+                "libntdb1"
+            ]
+        }
+    ],
+    "i386": [
+        {
+            "url": "http://archive.ubuntu.com/ubuntu";,
+            "dist": "vivid",
+            "component": "main",
+            "packages": [
+                "libsmbclient",
+                "samba-libs",
+                "libtalloc2",
+                "libtevent0",
+                "libwbclient0",
+                "libldb1",
+                "libntdb1"
+            ]
+        }
+    ]
+}

=== added file 'get-click-deps'
--- get-click-deps	1970-01-01 00:00:00 +0000
+++ get-click-deps	2015-09-07 10:44:39 +0000
@@ -0,0 +1,430 @@
+#!/usr/bin/python
+
+# Copyright (C) 2015 Stefano Verzegnassi <verzegnassi.stefano@xxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#
+# Source code at:
+# https://code.launchpad.net/~verzegnassi-stefano/+junk/get-click-deps
+#
+# A script to automate the fetching all the external dependencies of a
+# Click packaged application from internet, and copying them in a given
+# folder, so that the click packaging tool can easily build them in the
+# package.
+#
+# usage: get-click-deps [-h] [-f] [-d] [-e] [-c SCRIPT_PATH]
+#                       manifest_path {amd64,i386,armhf} target_path
+#
+# A tool for adding external libraries to a Ubuntu SDK application or scope.
+#
+# positional arguments:
+#   manifest_path         path of json file containing the list of packages to
+#                         be downloaded.
+#   {amd64,i386,armhf}    CPU architecture ("amd64", "i386" or "armhf")
+#   target_path           path to the target (click package or a build folder)
+#                         where this tool will include the downloaded binaries.
+#
+# optional arguments:
+#   -h, --help            show this help message and exit
+#   -f, --force-download  force a new download of the packages
+#   -d, --delete-temp     delete temp files at the end of the process
+#   -e, --extract-only    only create temp directory and extract the content of
+#                         downloaded packages
+# -c SCRIPT_PATH, --custom-script SCRIPT_PATH
+#                         run a custom script after the extraction of Debian
+#                         packages and before copying their content to the
+#                         target destination. The tool will execute the script
+#                         with the path to the packages dump as argument. If
+#                         the '-e' flag has been specified, the script will be
+#                         anyway executed. This option is useful when you need
+#                         to automatically modify the content of temp folder
+#                         (e.g. when you need to fix some path before including
+#                         the files in a click package).
+#
+#
+# USAGE EXAMPLE:
+# get-click-deps packages.json armhf <path/to/package.click>
+#
+# package.json is the package manifest, and contains all the references to the
+# .deb packages to be included into the Ubuntu SDK project.
+# If you're familiar to the Debian/Ubuntu world, you'll see that it's pretty
+# similar to therepository management of these distros:
+# see https://wiki.debian.org/SourcesList for further informations.
+#
+# An example of packages.json file is:
+# {
+#     "armhf": [
+#         {
+#             "url": "http://ports.ubuntu.com/ubuntu-ports/";,
+#             "dist": "vivid",
+#             "component": "main",
+#             "packages": [
+#                 "libgl1-mesa-glx",
+#                 "libxslt1.1",
+#                 "libxcb-glx0",
+#                 "libxcb-dri3-0",
+#                 "libxcb-present0",
+#                 "libxshmfence1",
+#                 "libxxf86vm1"
+#             ]
+#         },
+#
+#         {
+#             "url": "http://ppa.launchpad.net/canonical-community/ppa/ubuntu";,
+#             "dist": "vivid",
+#             "component": "main",
+#             "packages": [
+#                 "libreoffice-vanilla"
+#             ]
+#         }
+#     ],
+#
+#     "amd64": [
+#             "url": "http://ppa.launchpad.net/canonical-community/ppa/ubuntu";,
+#             "dist": "vivid",
+#             "component": "main",
+#             "packages": [
+#                 "libreoffice-vanilla"
+#             ]
+#         }
+#     ]
+# }
+#
+# Instead of a click package, you can specify the build folder used by Ubuntu
+# SDK for compiling the sources of your project. This way the binaries
+# downloaded by this tool will be automatically included in the .click package
+# the next time you ask Ubuntu SDK to create a new package.
+# Note that this feature is widely untested and still under development.
+#
+# KNOWN ISSUES:
+# - Not all the Debian packages contains the required files in a path that we
+#   can automatically copy into the Click package.
+#   For this reason it is suggested to run this tool with the '-e' flag at
+#   first, then to manually fix the paths in the temp folder.
+#   Then, you can run this tool without flags, so that it will use the temp
+#   folder it created with the previous running.
+#
+
+# TODO: Complete error handling
+# TODO: Make target_path optional if the '-e' flag has been specified.
+
+import sys
+import os
+import stat
+import time
+import argparse
+import json
+import gzip
+import subprocess
+
+
+def get_timestamp():
+    return time.time()
+
+
+def download_file(url, dest, verbose=True):
+    if verbose:
+        print "\nDownloading:\n%s" % url
+    os.system('cd %s && { curl -# -O %s ; cd - ; }' % (dest, url))
+
+
+def download_file_and_rename(url, dest, new_filename, verbose=True):
+    if verbose:
+        print "\nDownloading:\n%s" % url
+    new_path = os.path.join(dest, new_filename)
+    os.system('curl %s -# -o %s' % (url, new_path))
+
+    return new_path
+
+
+def get_package_download_url(package_name, packages_list, base_url):
+    pkgs_list = packages_list.decode('utf-8').split('\n')
+    index = pkgs_list.index('Package: %s' % package_name)
+
+    for i in range(index, len(pkgs_list)):
+        if pkgs_list[i].find('Filename:') > -1:
+            return "%s/%s" % (base_url, pkgs_list[i].replace('Filename: ', ''))
+
+
+def get_URLs_for_arch(manifest_path, arch, destpath):
+    urls = []
+
+    f = open(manifest_path)
+    content = json.load(f)
+    f.close()
+
+    # First of all, download repository index for each repository in the JSON
+    # package manifest.
+    repo_index = 0
+    for repo in content[arch]:
+        repo_index_url = '%s/dists/%s/%s/binary-%s/Packages.gz' % (
+            repo['url'],
+            repo['dist'],
+            repo['component'],
+            arch)
+
+        print "\nDownloading repository index at:\n%s" % repo_index_url
+
+        repo_index_zip = download_file_and_rename(
+            repo_index_url,
+            destpath,
+            'repo-index-%s-%s.gz' % (arch, repo_index),
+            False)
+
+        with gzip.open(repo_index_zip, 'r') as f:
+            repo_index_content = f.read()
+            f.close()
+
+        # Get the download URL of each package of the repository.
+        packages = repo['packages']
+        for package in packages:
+            urls.append(get_package_download_url(
+                package,
+                repo_index_content,
+                repo['url']))
+
+        repo_index += 1
+
+    print "\nObtained packages informations"
+    return urls
+
+
+def check_if_temp_folder_already_exists(path):
+    return os.path.isdir(path)
+
+
+def copy_directory_content(sourcepath, destpath):
+    os.system('cp -r %s/. %s' % (sourcepath, destpath))
+
+
+def delete_folder(path, recursive=False):
+    if recursive:
+        flag = '-rf'
+    else:
+        flag = '-f'
+    os.system('rm %s %s' % (flag, path))
+
+
+def extract_deb_package(deb_path, destpath):
+    os.system('dpkg-deb -x %s %s' % (deb_path, destpath))
+
+
+def extract_click_package(click_path, destpath):
+    extract_deb_package(click_path, destpath)
+
+    manifest = subprocess.check_output(['click', 'info', click_path])
+
+    # The manifest we get has an 'installed-size' key with the value of the
+    # previous package. Anyway this value will be replaced when we'll run
+    # 'click build <pkg>', so there's no reason for removing it here.
+    f = open(os.path.join(destpath, 'manifest.json'), 'w')
+    f.write(manifest)
+    f.close()
+
+    return destpath
+
+
+def build_click_package(source_dirpath):
+    output = subprocess.check_output(['click', 'build', source_dirpath])
+
+    for line in output.split(os.linesep):
+        if line.find('Successfully built package in ') > -1:
+            # FIXME: Very ugly.
+            path = line.replace('Successfully built package in \'', '')
+            path = path.replace('\'.', '')
+            return path
+
+
+def copy_file(sourcepath, destpath, overwrite=False):
+    flag = ''
+
+    if not overwrite:
+        flag = '-n'
+
+    os.system('cp %s %s %s' % (flag, sourcepath, destpath))
+
+
+# Argument parser
+parser = argparse.ArgumentParser(
+    description="A tool for adding external libraries to a Ubuntu SDK \
+        application or scope.")
+
+parser.add_argument(
+    '-f',
+    '--force-download',
+    dest='force_download',
+    action='store_true',
+    help='force a new download of the packages')
+
+parser.add_argument(
+    '-d',
+    '--delete-temp',
+    dest='delete_temp',
+    action='store_true',
+    help='delete temp files at the end of the process')
+
+parser.add_argument(
+    '-e',
+    '--extract-only',
+    dest='extract_only',
+    action='store_true',
+    help='only create temp directory and extract the content of downloaded \
+        packages')
+
+parser.add_argument(
+    '-c',
+    '--custom-script',
+    dest='script_path',
+    type=str,
+    help='run a custom script after the extraction of Debian packages and \
+        before copying their content to the target destination. The tool will \
+        execute the script with the path to the packages dump as argument. If \
+        the \'-e\' flag has been specified, the script will be anyway \
+        executed. This option is useful when you need to automatically modify \
+        the content of temp folder (e.g. when you need to fix some path \
+        before including the files in a click package).')
+
+parser.add_argument(
+    'manifest_path',
+    type=str,
+    help='path of json file containing the list of packages to be downloaded.')
+
+parser.add_argument(
+    'arch',
+    type=str,
+    choices=['amd64', 'i386', 'armhf'],
+    help='CPU architecture ("amd64", "i386" or "armhf")')
+
+parser.add_argument(
+    'target_path',
+    type=str,
+    help='path to the target (click package or a build folder) where this \
+        tool will include the downloaded binaries.')
+
+args = parser.parse_args()
+
+# Variables
+manifest_path = args.manifest_path
+target_path = args.target_path
+manifest_stat = os.stat(manifest_path)
+temp_folder = os.path.join(
+    '/tmp/',
+    'tmp-click-deps-%s-%s-%s-%s' % (
+        manifest_stat.st_dev,
+        manifest_stat.st_ino,
+        manifest_stat.st_size,
+        manifest_stat.st_mtime))
+temp_arch_folder = os.path.join(temp_folder, args.arch)
+is_click_target = os.path.isfile(args.target_path)
+
+# Check command line arguments
+if not os.path.isfile(manifest_path):
+    print "\n\nERROR: Package manifest is not a valid file. Exit..."
+    sys.exit()
+
+# Check if the target exists
+if not os.path.exists(target_path):
+    print "\n\nERROR: The specified target does not exist."
+    sys.exit()
+
+if os.path.isfile(target_path) and not target_path.endswith('.click'):
+    print "\n\nERROR: The specified target file is a valid .click package."
+    sys.exit()
+
+# Check if the script exist, if specified any.
+if args.script_path and not os.path.exists(args.script_path):
+    print "\n\nERROR: The specified script does not exists."
+    sys.exit()
+
+# If -f argument has been specified, remove all the existing data before
+# running this script.
+if args.force_download:
+    if os.path.isdir(temp_folder):
+        print "\nRemoving temp data of the previous run, as requested"
+        delete_folder(temp_folder, True)
+
+# Check if we already have run this script for the same target.
+if not check_if_temp_folder_already_exists(temp_folder):
+    # Create temp folder in /tmp
+    print "\nCreating temp folder in %s" % temp_folder
+    os.mkdir(temp_folder)
+    os.mkdir(temp_arch_folder)
+
+    # Parse the JSON package list and get the download URL of the packages.
+    debs_url_list = get_URLs_for_arch(
+        manifest_path,
+        args.arch,
+        temp_folder)
+
+    # Download packages from web
+    for url in debs_url_list:
+        download_file(url, temp_folder)
+
+    # Extract DEBs packages
+    print "\nExtracting .deb packages to %s" % temp_arch_folder
+    deb_pkgs_list = []
+    for file in os.listdir(temp_folder):
+        if file.endswith('.deb'):
+            deb_pkgs_list.append(file)
+
+    for deb_pkg in deb_pkgs_list:
+        extract_deb_package(
+            os.path.join(temp_folder, deb_pkg),
+            temp_arch_folder)
+else:
+    print "\nPackages are already downloaded. Use them..."
+
+# If a script has been specified, run it.
+if args.script_path:
+    print "\nRunning the script at: %s\n" % args.script_path
+
+    # Ensure that we can run the script, otherwise we don't have the permission
+    os.system('chmod +x %s' % args.script_path)
+    os.system('%s %s' % (args.script_path, temp_arch_folder))
+
+# If -e or --extract-only flags have been specified, we have completed our work
+if args.extract_only:
+    print "\n\nCompleted successfully."
+    sys.exit()
+
+# Copy temp_arch folder content to its destination
+if is_click_target:
+    print "\nExtracting target .click package"
+    temp_click_folder = extract_click_package(
+        args.target_path,
+        os.path.join('/tmp/', str(get_timestamp())))
+
+    print "\nAdding extracted binaries to the package"
+    copy_directory_content(temp_arch_folder, temp_click_folder)
+
+    new_click_package_path = build_click_package(temp_click_folder)
+    print "\nCreated new .click package at:\n%s" % new_click_package_path
+
+    print "\nReplacing older .click package"
+    copy_file(new_click_package_path, args.target_path, True)
+
+    delete_folder(temp_click_folder, True)
+
+else:
+    # TODO: This is still experimental and untested! May not work.
+    print "\nCopying extracted binaries to their destination"
+    copy_directory_content(temp_arch_folder, args.target_path)
+
+# Delete temp files
+if args.delete_temp:
+    print "\nRemoving temp files and directory, as requested"
+    delete_folder(temp_folder, True)
+
+# Exit
+print "\n\nCompleted successfully."