ubuntu-touch-coreapps-reviewers team mailing list archive
-
ubuntu-touch-coreapps-reviewers team
-
Mailing list archive
-
Message #04787
[Merge] lp:~popey/ubuntu-filemanager-app/add-click-deps into lp:ubuntu-filemanager-app
Alan Pope has proposed merging lp:~popey/ubuntu-filemanager-app/add-click-deps into lp:ubuntu-filemanager-app.
Commit message:
Add basics to cmake file and necessary parts to pull in samba libs.
Requested reviews:
Ubuntu File Manager Developers (ubuntu-filemanager-dev)
For more details, see:
https://code.launchpad.net/~popey/ubuntu-filemanager-app/add-click-deps/+merge/270287
First pass at adding get-click-deps from Stefano Verzegnassi to pull samba libs in during build process. Not final.
--
Your team Ubuntu File Manager Developers is requested to review the proposed merge of lp:~popey/ubuntu-filemanager-app/add-click-deps into lp:ubuntu-filemanager-app.
=== modified file 'CMakeLists.txt'
--- CMakeLists.txt 2014-09-08 10:45:41 +0000
+++ CMakeLists.txt 2015-09-07 10:24:38 +0000
@@ -33,6 +33,7 @@
OUTPUT_VARIABLE ARCH_TRIPLET
OUTPUT_STRIP_TRAILING_WHITESPACE
)
+
set(CMAKE_INSTALL_PREFIX /)
set(CMAKE_INSTALL_BINDIR /)
set(DATA_DIR /)
@@ -61,6 +62,15 @@
add_custom_target(com_ubuntu_calendar_CLICKFiles ALL SOURCES ${CLICK_FILES})
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/manifest.json filemanager.apparmor ${CONTENT_HUB_EXPORTER} DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+ MESSAGE("Grabbing upstream libs to ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs")
+ execute_process(
+ COMMAND mkdir ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs
+ COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/get-click-deps ${CMAKE_CURRENT_SOURCE_DIR}/filemanager-libs.json ${CLICK_ARCH} ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs
+ )
+ MESSAGE("Installing upstream libs from ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs/usr/lib/${ARCH_TRIPLET}/ to ${DATA_DIR}lib/${ARCH_TRIPLET}")
+ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/upstream-libs/usr/lib/${ARCH_TRIPLET}/ DESTINATION ${DATA_DIR}lib/${ARCH_TRIPLET})
+
else(CLICK_MODE)
execute_process(
COMMAND qmake -query QT_INSTALL_QML
=== added file 'filemanager-libs.json'
--- filemanager-libs.json 1970-01-01 00:00:00 +0000
+++ filemanager-libs.json 2015-09-07 10:24:38 +0000
@@ -0,0 +1,50 @@
+{
+ "armhf": [
+ {
+ "url": "http://ports.ubuntu.com/ubuntu-ports";,
+ "dist": "vivid",
+ "component": "main",
+ "packages": [
+ "libsmbclient",
+ "samba-libs",
+ "libtalloc2",
+ "libtevent0",
+ "libwbclient0",
+ "libldb1",
+ "libntdb1"
+ ]
+ }
+ ],
+ "amd64": [
+ {
+ "url": "http://archive.ubuntu.com/ubuntu";,
+ "dist": "vivid",
+ "component": "main",
+ "packages": [
+ "libsmbclient",
+ "samba-libs",
+ "libtalloc2",
+ "libtevent0",
+ "libwbclient0",
+ "libldb1",
+ "libntdb1"
+ ]
+ }
+ ],
+ "i386": [
+ {
+ "url": "http://archive.ubuntu.com/ubuntu";,
+ "dist": "vivid",
+ "component": "main",
+ "packages": [
+ "libsmbclient",
+ "samba-libs",
+ "libtalloc2",
+ "libtevent0",
+ "libwbclient0",
+ "libldb1",
+ "libntdb1"
+ ]
+ }
+ ]
+}
=== added file 'get-click-deps'
--- get-click-deps 1970-01-01 00:00:00 +0000
+++ get-click-deps 2015-09-07 10:24:38 +0000
@@ -0,0 +1,430 @@
+#!/usr/bin/python
+
+# Copyright (C) 2015 Stefano Verzegnassi <verzegnassi.stefano@xxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#
+# Source code at:
+# https://code.launchpad.net/~verzegnassi-stefano/+junk/get-click-deps
+#
+# A script to automate the fetching all the external dependencies of a
+# Click packaged application from internet, and copying them in a given
+# folder, so that the click packaging tool can easily build them in the
+# package.
+#
+# usage: get-click-deps [-h] [-f] [-d] [-e] [-c SCRIPT_PATH]
+# manifest_path {amd64,i386,armhf} target_path
+#
+# A tool for adding external libraries to a Ubuntu SDK application or scope.
+#
+# positional arguments:
+# manifest_path path of json file containing the list of packages to
+# be downloaded.
+# {amd64,i386,armhf} CPU architecture ("amd64", "i386" or "armhf")
+# target_path path to the target (click package or a build folder)
+# where this tool will include the downloaded binaries.
+#
+# optional arguments:
+# -h, --help show this help message and exit
+# -f, --force-download force a new download of the packages
+# -d, --delete-temp delete temp files at the end of the process
+# -e, --extract-only only create temp directory and extract the content of
+# downloaded packages
+# -c SCRIPT_PATH, --custom-script SCRIPT_PATH
+# run a custom script after the extraction of Debian
+# packages and before copying their content to the
+# target destination. The tool will execute the script
+# with the path to the packages dump as argument. If
+# the '-e' flag has been specified, the script will be
+# anyway executed. This option is useful when you need
+# to automatically modify the content of temp folder
+# (e.g. when you need to fix some path before including
+# the files in a click package).
+#
+#
+# USAGE EXAMPLE:
+# get-click-deps packages.json armhf <path/to/package.click>
+#
+# package.json is the package manifest, and contains all the references to the
+# .deb packages to be included into the Ubuntu SDK project.
+# If you're familiar to the Debian/Ubuntu world, you'll see that it's pretty
+# similar to therepository management of these distros:
+# see https://wiki.debian.org/SourcesList for further informations.
+#
+# An example of packages.json file is:
+# {
+# "armhf": [
+# {
+# "url": "http://ports.ubuntu.com/ubuntu-ports/";,
+# "dist": "vivid",
+# "component": "main",
+# "packages": [
+# "libgl1-mesa-glx",
+# "libxslt1.1",
+# "libxcb-glx0",
+# "libxcb-dri3-0",
+# "libxcb-present0",
+# "libxshmfence1",
+# "libxxf86vm1"
+# ]
+# },
+#
+# {
+# "url": "http://ppa.launchpad.net/canonical-community/ppa/ubuntu";,
+# "dist": "vivid",
+# "component": "main",
+# "packages": [
+# "libreoffice-vanilla"
+# ]
+# }
+# ],
+#
+# "amd64": [
+# "url": "http://ppa.launchpad.net/canonical-community/ppa/ubuntu";,
+# "dist": "vivid",
+# "component": "main",
+# "packages": [
+# "libreoffice-vanilla"
+# ]
+# }
+# ]
+# }
+#
+# Instead of a click package, you can specify the build folder used by Ubuntu
+# SDK for compiling the sources of your project. This way the binaries
+# downloaded by this tool will be automatically included in the .click package
+# the next time you ask Ubuntu SDK to create a new package.
+# Note that this feature is widely untested and still under development.
+#
+# KNOWN ISSUES:
+# - Not all the Debian packages contains the required files in a path that we
+# can automatically copy into the Click package.
+# For this reason it is suggested to run this tool with the '-e' flag at
+# first, then to manually fix the paths in the temp folder.
+# Then, you can run this tool without flags, so that it will use the temp
+# folder it created with the previous running.
+#
+
+# TODO: Complete error handling
+# TODO: Make target_path optional if the '-e' flag has been specified.
+
+import sys
+import os
+import stat
+import time
+import argparse
+import json
+import gzip
+import subprocess
+
+
+def get_timestamp():
+ return time.time()
+
+
+def download_file(url, dest, verbose=True):
+ if verbose:
+ print "\nDownloading:\n%s" % url
+ os.system('cd %s && { curl -# -O %s ; cd - ; }' % (dest, url))
+
+
+def download_file_and_rename(url, dest, new_filename, verbose=True):
+ if verbose:
+ print "\nDownloading:\n%s" % url
+ new_path = os.path.join(dest, new_filename)
+ os.system('curl %s -# -o %s' % (url, new_path))
+
+ return new_path
+
+
+def get_package_download_url(package_name, packages_list, base_url):
+ pkgs_list = packages_list.decode('utf-8').split('\n')
+ index = pkgs_list.index('Package: %s' % package_name)
+
+ for i in range(index, len(pkgs_list)):
+ if pkgs_list[i].find('Filename:') > -1:
+ return "%s/%s" % (base_url, pkgs_list[i].replace('Filename: ', ''))
+
+
+def get_URLs_for_arch(manifest_path, arch, destpath):
+ urls = []
+
+ f = open(manifest_path)
+ content = json.load(f)
+ f.close()
+
+ # First of all, download repository index for each repository in the JSON
+ # package manifest.
+ repo_index = 0
+ for repo in content[arch]:
+ repo_index_url = '%s/dists/%s/%s/binary-%s/Packages.gz' % (
+ repo['url'],
+ repo['dist'],
+ repo['component'],
+ arch)
+
+ print "\nDownloading repository index at:\n%s" % repo_index_url
+
+ repo_index_zip = download_file_and_rename(
+ repo_index_url,
+ destpath,
+ 'repo-index-%s-%s.gz' % (arch, repo_index),
+ False)
+
+ with gzip.open(repo_index_zip, 'r') as f:
+ repo_index_content = f.read()
+ f.close()
+
+ # Get the download URL of each package of the repository.
+ packages = repo['packages']
+ for package in packages:
+ urls.append(get_package_download_url(
+ package,
+ repo_index_content,
+ repo['url']))
+
+ repo_index += 1
+
+ print "\nObtained packages informations"
+ return urls
+
+
+def check_if_temp_folder_already_exists(path):
+ return os.path.isdir(path)
+
+
+def copy_directory_content(sourcepath, destpath):
+ os.system('cp -r %s/. %s' % (sourcepath, destpath))
+
+
+def delete_folder(path, recursive=False):
+ if recursive:
+ flag = '-rf'
+ else:
+ flag = '-f'
+ os.system('rm %s %s' % (flag, path))
+
+
+def extract_deb_package(deb_path, destpath):
+ os.system('dpkg-deb -x %s %s' % (deb_path, destpath))
+
+
+def extract_click_package(click_path, destpath):
+ extract_deb_package(click_path, destpath)
+
+ manifest = subprocess.check_output(['click', 'info', click_path])
+
+ # The manifest we get has an 'installed-size' key with the value of the
+ # previous package. Anyway this value will be replaced when we'll run
+ # 'click build <pkg>', so there's no reason for removing it here.
+ f = open(os.path.join(destpath, 'manifest.json'), 'w')
+ f.write(manifest)
+ f.close()
+
+ return destpath
+
+
+def build_click_package(source_dirpath):
+ output = subprocess.check_output(['click', 'build', source_dirpath])
+
+ for line in output.split(os.linesep):
+ if line.find('Successfully built package in ') > -1:
+ # FIXME: Very ugly.
+ path = line.replace('Successfully built package in \'', '')
+ path = path.replace('\'.', '')
+ return path
+
+
+def copy_file(sourcepath, destpath, overwrite=False):
+ flag = ''
+
+ if not overwrite:
+ flag = '-n'
+
+ os.system('cp %s %s %s' % (flag, sourcepath, destpath))
+
+
+# Argument parser
+parser = argparse.ArgumentParser(
+ description="A tool for adding external libraries to a Ubuntu SDK \
+ application or scope.")
+
+parser.add_argument(
+ '-f',
+ '--force-download',
+ dest='force_download',
+ action='store_true',
+ help='force a new download of the packages')
+
+parser.add_argument(
+ '-d',
+ '--delete-temp',
+ dest='delete_temp',
+ action='store_true',
+ help='delete temp files at the end of the process')
+
+parser.add_argument(
+ '-e',
+ '--extract-only',
+ dest='extract_only',
+ action='store_true',
+ help='only create temp directory and extract the content of downloaded \
+ packages')
+
+parser.add_argument(
+ '-c',
+ '--custom-script',
+ dest='script_path',
+ type=str,
+ help='run a custom script after the extraction of Debian packages and \
+ before copying their content to the target destination. The tool will \
+ execute the script with the path to the packages dump as argument. If \
+ the \'-e\' flag has been specified, the script will be anyway \
+ executed. This option is useful when you need to automatically modify \
+ the content of temp folder (e.g. when you need to fix some path \
+ before including the files in a click package).')
+
+parser.add_argument(
+ 'manifest_path',
+ type=str,
+ help='path of json file containing the list of packages to be downloaded.')
+
+parser.add_argument(
+ 'arch',
+ type=str,
+ choices=['amd64', 'i386', 'armhf'],
+ help='CPU architecture ("amd64", "i386" or "armhf")')
+
+parser.add_argument(
+ 'target_path',
+ type=str,
+ help='path to the target (click package or a build folder) where this \
+ tool will include the downloaded binaries.')
+
+args = parser.parse_args()
+
+# Variables
+manifest_path = args.manifest_path
+target_path = args.target_path
+manifest_stat = os.stat(manifest_path)
+temp_folder = os.path.join(
+ '/tmp/',
+ 'tmp-click-deps-%s-%s-%s-%s' % (
+ manifest_stat.st_dev,
+ manifest_stat.st_ino,
+ manifest_stat.st_size,
+ manifest_stat.st_mtime))
+temp_arch_folder = os.path.join(temp_folder, args.arch)
+is_click_target = os.path.isfile(args.target_path)
+
+# Check command line arguments
+if not os.path.isfile(manifest_path):
+ print "\n\nERROR: Package manifest is not a valid file. Exit..."
+ sys.exit()
+
+# Check if the target exists
+if not os.path.exists(target_path):
+ print "\n\nERROR: The specified target does not exist."
+ sys.exit()
+
+if os.path.isfile(target_path) and not target_path.endswith('.click'):
+ print "\n\nERROR: The specified target file is a valid .click package."
+ sys.exit()
+
+# Check if the script exist, if specified any.
+if args.script_path and not os.path.exists(args.script_path):
+ print "\n\nERROR: The specified script does not exists."
+ sys.exit()
+
+# If -f argument has been specified, remove all the existing data before
+# running this script.
+if args.force_download:
+ if os.path.isdir(temp_folder):
+ print "\nRemoving temp data of the previous run, as requested"
+ delete_folder(temp_folder, True)
+
+# Check if we already have run this script for the same target.
+if not check_if_temp_folder_already_exists(temp_folder):
+ # Create temp folder in /tmp
+ print "\nCreating temp folder in %s" % temp_folder
+ os.mkdir(temp_folder)
+ os.mkdir(temp_arch_folder)
+
+ # Parse the JSON package list and get the download URL of the packages.
+ debs_url_list = get_URLs_for_arch(
+ manifest_path,
+ args.arch,
+ temp_folder)
+
+ # Download packages from web
+ for url in debs_url_list:
+ download_file(url, temp_folder)
+
+ # Extract DEBs packages
+ print "\nExtracting .deb packages to %s" % temp_arch_folder
+ deb_pkgs_list = []
+ for file in os.listdir(temp_folder):
+ if file.endswith('.deb'):
+ deb_pkgs_list.append(file)
+
+ for deb_pkg in deb_pkgs_list:
+ extract_deb_package(
+ os.path.join(temp_folder, deb_pkg),
+ temp_arch_folder)
+else:
+ print "\nPackages are already downloaded. Use them..."
+
+# If a script has been specified, run it.
+if args.script_path:
+ print "\nRunning the script at: %s\n" % args.script_path
+
+ # Ensure that we can run the script, otherwise we don't have the permission
+ os.system('chmod +x %s' % args.script_path)
+ os.system('%s %s' % (args.script_path, temp_arch_folder))
+
+# If -e or --extract-only flags have been specified, we have completed our work
+if args.extract_only:
+ print "\n\nCompleted successfully."
+ sys.exit()
+
+# Copy temp_arch folder content to its destination
+if is_click_target:
+ print "\nExtracting target .click package"
+ temp_click_folder = extract_click_package(
+ args.target_path,
+ os.path.join('/tmp/', str(get_timestamp())))
+
+ print "\nAdding extracted binaries to the package"
+ copy_directory_content(temp_arch_folder, temp_click_folder)
+
+ new_click_package_path = build_click_package(temp_click_folder)
+ print "\nCreated new .click package at:\n%s" % new_click_package_path
+
+ print "\nReplacing older .click package"
+ copy_file(new_click_package_path, args.target_path, True)
+
+ delete_folder(temp_click_folder, True)
+
+else:
+ # TODO: This is still experimental and untested! May not work.
+ print "\nCopying extracted binaries to their destination"
+ copy_directory_content(temp_arch_folder, args.target_path)
+
+# Delete temp files
+if args.delete_temp:
+ print "\nRemoving temp files and directory, as requested"
+ delete_folder(temp_folder, True)
+
+# Exit
+print "\n\nCompleted successfully."
References
