-
Notifications
You must be signed in to change notification settings - Fork 11
/
fetch_dependencies.py
164 lines (138 loc) · 6.79 KB
/
fetch_dependencies.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
#! /usr/bin/python
#
# Simple script to update a set of common directories that are needed as dependencies of the current project
# Usage:
# FetchDependencies.py [latest]
#
# If "latest" is specified, the latest commit will be checked out.
# Otherwise, the repos will be updated to the commit specified in the "gitMapping" table.
# If the required commit in the "gitMapping" is None, the repo will be updated to the latest commit.
import os
import subprocess
import sys
import urllib
import zipfile
import tarfile
import platform
import shutil
#parser = argparse.ArgumentParser(description='Fetch external dependencies for Brotli-G')
#parser.add_argument('--enable_brotlig', action='store_true', help='Enable fetching dependencies for Brotli-G support')
#args = parser.parse_args()
isPython3OrAbove = None
if sys.version_info[0] >= 3:
isPython3OrAbove = True
if isPython3OrAbove:
import urllib.request
# to allow the script to be run from anywhere - not just the cwd - store the absolute path to the script file
scriptRoot = os.path.dirname(os.path.realpath(__file__))
# root is one folders up from scriptRoot (brotli_g_sdk/build)
BrotliGRoot = scriptRoot
# When running this script on Windows (and not under cygwin), we need to set the shell=True argument to Popen and similar calls
# Without this option, Jenkins builds fail to find the correct version of git
SHELLARG = False
if ( sys.platform.startswith("win32")):
# running on windows under default shell
SHELLARG = True
# Print the version of git being used. This also confirms that the script can find git
try:
subprocess.call(["git","--version"], shell=SHELLARG)
except OSError:
# likely to be due to inability to find git command
print("Error calling command: git --version")
# Calculate the root of the git server - all dependencies should be retrieved from the same server
gitURL = subprocess.check_output(["git", "-C", scriptRoot, "remote", "get-url", "origin"], shell=SHELLARG)
# Strip everything after the last '/' from the URL to retrieve the root
if gitURL == None:
print("Error: Unable to determine origin for Brotli-G git project")
exit(1)
gitUrlStr = (str(gitURL).lstrip("b'"))
gitRoot = (gitUrlStr.rsplit('/',1))[0] + '/'
# If cloning from github - use the master branch - otherwise use amd-master
if "github" in gitUrlStr:
gitBranch = "master"
print("\nFetching dependencies from: " + gitRoot + " - using branch: " + gitBranch)
# Define a set of dependencies that exist as separate git projects. The parameters are:
# "git repo name" : ["Directory for clone relative to this script", "branch or commit to checkout (or None for top of tree)"
# External lib location
externalLoc = "external/"
# Sample external lib location
sampleExternalLoc = "sample/external/"
# source reviews for use
# "https://github.com/google/brotli.git" : [externalLoc+"brotli", "1.0.9"]
# Libs to build
gitMapping = {
"https://github.com/google/brotli.git" : [externalLoc+"brotli", "v1.0.9"],
"https://github.com/microsoft/DirectX-Headers.git" : [sampleExternalLoc+"DirectX-Headers", "v1.611.0"]
}
downloadMapping = {
"https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2112/dxc_2021_12_08.zip" : sampleExternalLoc+"dxc_2021_12_08/"
}
userAgentString = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246"
# for each dependency - test if it has already been fetched - if not, then fetch it, otherwise update it to top of tree
def downloadgit(key, path, reqdCommit):
# Target path, relative to workspace
path = gitMapping[key][0]
source = key
reqdCommit = gitMapping[key][1]
# reqdCommit may be "None" - or user may override commit via command line. In this case, use tip of tree
if((len(sys.argv) != 1 and sys.argv[1] == "latest") or reqdCommit is None):
reqdCommit = gitBranch
print("\nChecking out commit: " + reqdCommit + " for " + key)
os.chdir(BrotliGRoot)
if os.path.isdir(path):
# directory exists - get latest from git using pull
print("Directory " + path + " exists. \n\tUsing 'git fetch' to get latest from " + source)
sys.stdout.flush()
try:
subprocess.check_call(["git", "-C", path, "fetch", "origin"], shell=SHELLARG)
except subprocess.CalledProcessError as e:
print ("'git fetch' failed with return code: %d\n" % e.returncode)
try:
subprocess.check_call(["git", "-C", path, "checkout", reqdCommit], shell=SHELLARG)
except subprocess.CalledProcessError as e:
print ("'git checkout' failed with return code: %d\n" % e.returncode)
sys.stderr.flush()
sys.stdout.flush()
else:
# directory doesn't exist - clone from git
print("Directory " + path + " does not exist. \n\tUsing 'git clone' to get latest from " + source)
sys.stdout.flush()
try:
subprocess.check_call(["git", "clone", source, path], shell=SHELLARG)
subprocess.check_call(["git", "-C", path, "checkout", reqdCommit], shell=SHELLARG)
except subprocess.CalledProcessError as e:
print ("'git clone' failed with return code: %d\n" % e.returncode)
sys.exit(1)
sys.stderr.flush()
sys.stdout.flush()
for key in gitMapping:
downloadgit(key, gitMapping[key][0], gitMapping[key][1])
# routine for downloading and unzipping an archive
def downloadandunzip(key, value):
# convert targetPath to OS specific format
tmppath = os.path.join(scriptRoot, "", value)
# clean up path, collapsing any ../ and converting / to \ for Windows
targetPath = os.path.normpath(tmppath)
if False == os.path.isdir(targetPath):
os.makedirs(targetPath)
zipfileName = key.split('/')[-1].split('#')[0].split('?')[0]
zipPath = os.path.join(targetPath, zipfileName)
if False == os.path.isfile(zipPath):
print("\nDownloading " + key + " into " + zipPath)
if isPython3OrAbove:
req = urllib.request.Request(key, headers={"User-Agent": userAgentString})
with urllib.request.urlopen(req) as response:
with open(zipPath, 'wb') as outputFile:
shutil.copyfileobj(response, outputFile)
else:
urllib.urlretrieve(key, zipPath)
if os.path.splitext(zipPath)[1] == ".zip":
zipfile.ZipFile(zipPath).extractall(targetPath)
os.remove(zipPath)
elif os.path.splitext(zipPath)[1] == ".gz":
tarfile.open(zipPath).extractall(targetPath)
os.remove(zipPath)
# for each archived release, download and unzip the artifacts into the target location
for key in downloadMapping:
downloadandunzip(key, downloadMapping[key])
print("Fetching dependencies finished successfully.\n")