diff --git a/Meta/gn/build/download_cache.gni b/Meta/gn/build/download_cache.gni new file mode 100644 index 0000000000..c923858e63 --- /dev/null +++ b/Meta/gn/build/download_cache.gni @@ -0,0 +1,4 @@ +declare_args() { + # Location of shared cache of downloaded files + cache_path = "$root_gen_dir/Cache/" +} diff --git a/Meta/gn/build/download_file.gni b/Meta/gn/build/download_file.gni new file mode 100644 index 0000000000..a05a91fb70 --- /dev/null +++ b/Meta/gn/build/download_file.gni @@ -0,0 +1,80 @@ +# +# This file introduces a template for calling download_file.py +# +# download_file behaves like CMake's file(DOWNLOAD) with the addtion +# of version checking the file against a build system defined version. +# +# Parameters: +# url (required) [string] +# +# output (required) [string] +# +# version (required) [string] +# Version of the file for caching purposes +# +# version_file (reqiured) [string] +# Filename to write the version to in the filesystem +# +# cache [String] +# Directory to clear on version mismatch +# +# +# Example use: +# +# download_file("my_tarball") { +# url = "http://example.com/xyz.tar.gz" +# output = "$root_gen_dir/MyModule/xyz.tar.gz" +# version = "1.2.3" +# version_file = "$root_gen_dir/MyModule/xyz_version.txt" +# } +# + +template("download_file") { + assert(defined(invoker.url), "must set 'url' in $target_name") + assert(defined(invoker.output), "must set 'output' in $target_name") + assert(defined(invoker.version), "must set 'version' in $target_name") + assert(defined(invoker.version_file), + "must set 'version_file' in $target_name") + + action(target_name) { + script = "//Meta/gn/build/download_file.py" + + sources = [] + if (defined(invoker.cache)) { + outputs = [ + invoker.cache + "/" + invoker.output, + invoker.cache + "/" + invoker.version_file, + ] + } else { + outputs = [ + invoker.output, + invoker.version_file, + ] + } + args = [ + "-o", + rebase_path(outputs[0], root_build_dir), + "-f", + rebase_path(outputs[1], root_build_dir), + "-v", + invoker.version, + invoker.url, + ] + if (defined(invoker.cache)) { + args += [ + "-c", + rebase_path(invoker.cache, root_build_dir), + ] + } + + forward_variables_from(invoker, + [ + "configs", + "deps", + "public_configs", + "public_deps", + "testonly", + "visibility", + ]) + } +} diff --git a/Meta/gn/build/download_file.py b/Meta/gn/build/download_file.py new file mode 100644 index 0000000000..ffeef7c411 --- /dev/null +++ b/Meta/gn/build/download_file.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +r"""Downloads a file as a build artifact. + +The file is downloaded to the specified directory. + +It's intended to be used for files that are cached between runs. + +""" + +import argparse +import os +import pathlib +import shutil +import sys +import tempfile +import urllib.request + + +def main(): + parser = argparse.ArgumentParser( + epilog=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('url', help='input url') + parser.add_argument('-o', '--output', required=True, + help='output file') + parser.add_argument('-v', '--version', required=True, + help='version of file to detect mismatches and redownload') + parser.add_argument('-f', '--version-file', required=True, + help='filesystem location to cache version') + parser.add_argument('-c', "--cache-path", required=False, + help='path for cached files to clear on version mismatch') + args = parser.parse_args() + + version_from_file = '' + version_file = pathlib.Path(args.version_file) + if version_file.exists(): + with version_file.open('r') as f: + version_from_file = f.readline().strip() + + if version_from_file == args.version: + return 0 + + # Fresh build or version mismatch, delete old cache + if (args.cache_path): + cache_path = pathlib.Path(args.cache_path) + shutil.rmtree(cache_path, ignore_errors=True) + cache_path.mkdir(parents=True) + + print(f"Downloading version {args.version} of {args.output}...", end='') + + with urllib.request.urlopen(args.url) as f: + try: + with tempfile.NamedTemporaryFile(delete=False, + dir=pathlib.Path(args.output).parent) as out: + out.write(f.read()) + os.rename(out.name, args.output) + except IOError: + os.unlink(out.name) + + print("done") + + with open(version_file, 'w') as f: + f.write(args.version) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/Meta/gn/build/extract_archive_contents.gni b/Meta/gn/build/extract_archive_contents.gni new file mode 100644 index 0000000000..e000b0aee5 --- /dev/null +++ b/Meta/gn/build/extract_archive_contents.gni @@ -0,0 +1,76 @@ +# +# This file introduces templates for calling extract_archive_contents.py +# +# extract_archive_contents.py behaves like CMake's file(ARCHIVE_EXTRACT) +# +# Parameters: +# archive (required) [string] +# +# files (required) [list of strings] +# Relative paths to the root of the archive of files to extract +# +# directory (required) [string] +# Output directory root for all the files +# +# paths (optional) [list of strings] +# Relative paths to the root of the archive of directories to extract +# +# Example use: +# +# extract_archive_contents("my_files") { +# archive = "$root_gen_dir/MyModule/xyz.tar.gz" +# directory = "$root_gen_dir/MyModule" +# files = [ +# "file_one.txt", +# "file_two" +# ] +# paths = [ "some_dir" ] +# } +# + +template("extract_archive_contents") { + assert(defined(invoker.archive), "must set 'archive' in $target_name") + assert(defined(invoker.files) || defined(invoker.paths), + "must set 'files' and/or 'paths' in $target_name") + assert(defined(invoker.directory), "must set 'directory' in $target_name") + + action(target_name) { + script = "//Meta/gn/build/extract_archive_contents.py" + + paths = [] + if (defined(invoker.paths)) { + foreach(path, invoker.paths) { + paths += [ path + "/" ] + } + } + files = [] + if (defined(invoker.files)) { + files = invoker.files + } + stamp_file = invoker.directory + "$target_name.stamp" + + sources = invoker.archive + outputs = [] + args = [ + "-d", + rebase_path(invoker.directory, root_build_dir), + "-s", + rebase_path(stamp_file, root_build_dir), + rebase_path(sources[0], root_build_dir), + ] + files + paths + foreach(file, files) { + outputs += [ invoker.directory + file ] + } + outputs += [ stamp_file ] + + forward_variables_from(invoker, + [ + "configs", + "deps", + "public_configs", + "public_deps", + "testonly", + "visibility", + ]) + } +} diff --git a/Meta/gn/build/extract_archive_contents.py b/Meta/gn/build/extract_archive_contents.py new file mode 100644 index 0000000000..10b5b930f2 --- /dev/null +++ b/Meta/gn/build/extract_archive_contents.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +r"""Extracts files from an archive for use in the build + +It's intended to be used for files that are cached between runs. + +""" + +import argparse +import pathlib +import tarfile +import zipfile +import sys + + +def extract_member(file, destination, path): + """ + Extract a single file from a ZipFile or TarFile + + :param ZipFile|TarFile file: Archive object to extract from + :param Path destination: Location to write the file + :param str path: Filename to extract from archive. + """ + destination_path = destination / path + if destination_path.exists(): + return + destination_path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(file, tarfile.TarFile): + with file.extractfile(path) as member: + destination_path.write_text(member.read().decode('utf-8')) + else: + assert isinstance(file, zipfile.ZipFile) + with file.open(path) as member: + destination_path.write_text(member.read().decode('utf-8')) + + +def extract_directory(file, destination, path): + """ + Extract a directory from a ZipFile or TarFile + + :param ZipFile|TarFile file: Archive object to extract from + :param Path destination: Location to write the files + :param str path: Directory name to extract from archive. + """ + destination_path = destination / path + if destination_path.exists(): + return + destination_path.mkdir(parents=True, exist_ok=True) + if not isinstance(file, zipfile.ZipFile): + raise NotImplementedError + # FIXME: This loops over the entire archive len(args.paths) times. Decrease complexity + for entry in file.namelist(): + if entry.startswith(path): + entry_destination = destination / entry + if entry.endswith('/'): + entry_destination.mkdir(exist_ok=True) + continue + with file.open(entry) as member: + entry_destination.write_text(member.read().decode('utf-8')) + + +def main(): + parser = argparse.ArgumentParser( + epilog=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('archive', help='input archive') + parser.add_argument('paths', nargs='*', help='paths to extract from the archive') + parser.add_argument('-s', "--stamp", required=False, + help='stamp file name to create after operation is done') + parser.add_argument('-d', "--destination", required=True, + help='directory to write the extracted file to') + args = parser.parse_args() + + archive = pathlib.Path(args.archive) + destination = pathlib.Path(args.destination) + + def extract_paths(file, paths): + for path in paths: + if path.endswith('/'): + extract_directory(file, destination, path) + else: + extract_member(file, destination, path) + + if tarfile.is_tarfile(archive): + with tarfile.open(archive) as f: + extract_paths(f, args.paths) + elif zipfile.is_zipfile(archive): + with zipfile.ZipFile(archive) as f: + extract_paths(f, args.paths) + else: + print(f"Unknown file type for {archive}, unable to extract {args.path}") + return 1 + + if args.stamp: + pathlib.Path(args.stamp).touch() + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/Meta/gn/secondary/BUILD.gn b/Meta/gn/secondary/BUILD.gn index 138e26abcd..ff223d5d41 100644 --- a/Meta/gn/secondary/BUILD.gn +++ b/Meta/gn/secondary/BUILD.gn @@ -4,6 +4,7 @@ group("default") { deps = [ "//Meta/Lagom/Tools/CodeGenerators/IPCCompiler", "//Tests", + "//Userland/Libraries/LibTimeZone", ] testonly = true } diff --git a/Meta/gn/secondary/Userland/Libraries/LibTimeZone/BUILD.gn b/Meta/gn/secondary/Userland/Libraries/LibTimeZone/BUILD.gn index f14e966cfb..7ff9a3c044 100644 --- a/Meta/gn/secondary/Userland/Libraries/LibTimeZone/BUILD.gn +++ b/Meta/gn/secondary/Userland/Libraries/LibTimeZone/BUILD.gn @@ -1,7 +1,44 @@ +import("//Meta/gn/build/compiled_action.gni") +import("//Meta/gn/build/download_cache.gni") +import("//Meta/gn/build/download_file.gni") +import("//Meta/gn/build/extract_archive_contents.gni") + declare_args() { # If true, Download tzdata from data.iana.org and use it in LibTimeZone # Data will be downloaded to $cache_path/TZDB - enable_timezone_database_download = false + enable_timezone_database_download = true +} + +tzdb_cache = cache_path + "TZDB/" + +if (enable_timezone_database_download) { + download_file("timezone_database_download") { + version = "2023c" + url = + "https://data.iana.org/time-zones/releases/tzdata" + version + ".tar.gz" + cache = tzdb_cache + output = "tzdb.tar.gz" + version_file = "version.txt" + } + extract_archive_contents("timezone_database_files") { + deps = [ ":timezone_database_download" ] + archive = get_target_outputs(":timezone_database_download") + directory = tzdb_cache + + # NOSORT + files = [ + "zone1970.tab", + "africa", + "antarctica", + "asia", + "australasia", + "backward", + "etcetera", + "europe", + "northamerica", + "southamerica", + ] + } } source_set("LibTimeZone") { @@ -17,6 +54,6 @@ source_set("LibTimeZone") { "//Userland/Libraries/LibCore", ] if (enable_timezone_database_download) { - deps += [ ":timezone_data" ] + deps += [ ":timezone_database_files" ] } }