33
44"""Use the GitHub API to download built artifacts."""
55
6+ import collections
67import datetime
7- import json
8+ import fnmatch
9+ import operator
810import os
911import os .path
1012import sys
1315
1416import requests
1517
18+
1619def download_url (url , filename ):
1720 """Download a file from `url` to `filename`."""
1821 response = requests .get (url , stream = True )
@@ -23,6 +26,7 @@ def download_url(url, filename):
2326 else :
2427 raise RuntimeError (f"Fetching { url } { response .status_code } )
2528
29+
2630def unpack_zipfile (filename ):
2731 """Unpack a zipfile, using the names in the zip."""
2832 with open (filename , "rb" ) as fzip :
@@ -31,8 +35,10 @@ def unpack_zipfile(filename):
3135 print (f" extracting { name } )
3236 z .extract (name )
3337
38+
3439def utc2local (timestring ):
35- """Convert a UTC time into local time in a more readable form.
40+ """
41+ Convert a UTC time into local time in a more readable form.
3642
3743 For example: '20201208T122900Z' to '2020-12-08 07:29:00'.
3844
@@ -44,25 +50,65 @@ def utc2local(timestring):
4450 local = utc + offset
4551 return local .strftime ("%Y-%m-%d %H:%M:%S" )
4652
47- dest = "dist"
48- repo_owner = sys .argv [1 ]
49- temp_zip = "artifacts.zip"
5053
51- os .makedirs (dest , exist_ok = True )
52- os .chdir (dest )
54+ def all_items (url , key ):
55+ """
56+ Get all items from a paginated GitHub URL.
5357
54- r = requests .get (f"https://api.github.com/repos/{ repo_owner } )
55- if r .status_code == 200 :
56- dists = [a for a in r .json ()["artifacts" ] if a ["name" ] == "dist" ]
57- if not dists :
58- print ("No recent dists!" )
59- else :
60- latest = max (dists , key = lambda a : a ["created_at" ])
61- print (f"Artifacts created at { utc2local (latest ['created_at' ])} )
62- download_url (latest ["archive_download_url" ], temp_zip )
58+ `key` is the key in the top-level returned object that has a list of items.
59+
60+ """
61+ url += ("&" if "?" in url else "?" ) + "per_page=100"
62+ while url :
63+ response = requests .get (url )
64+ response .raise_for_status ()
65+ data = response .json ()
66+ if isinstance (data , dict ) and (msg := data .get ("message" )):
67+ raise RuntimeError (f"URL { url !r} { msg } )
68+ yield from data .get (key , ())
69+ try :
70+ url = response .links .get ("next" ).get ("url" )
71+ except AttributeError :
72+ url = None
73+
74+
75+ def main (owner_repo , artifact_pattern , dest_dir ):
76+ """
77+ Download and unzip the latest artifacts matching a pattern.
78+
79+ `owner_repo` is a GitHub pair for the repo, like "nedbat/coveragepy".
80+ `artifact_pattern` is a filename glob for the artifact name.
81+ `dest_dir` is the directory to unpack them into.
82+
83+ """
84+ # Get all artifacts matching the pattern, grouped by name.
85+ url = f"https://api.github.com/repos/{ owner_repo }
86+ artifacts_by_name = collections .defaultdict (list )
87+ for artifact in all_items (url , "artifacts" ):
88+ name = artifact ["name" ]
89+ if not fnmatch .fnmatch (name , artifact_pattern ):
90+ continue
91+ artifacts_by_name [name ].append (artifact )
92+
93+ os .makedirs (dest_dir , exist_ok = True )
94+ os .chdir (dest_dir )
95+ temp_zip = "artifacts.zip"
96+
97+ # Download the latest of each name.
98+ # I'd like to use created_at, because it seems like the better value to use,
99+ # but it is in the wrong time zone, and updated_at is the same but correct.
100+ # Bug report here: https://github.com/actions/upload-artifact/issues/488.
101+ for name , artifacts in artifacts_by_name .items ():
102+ artifact = max (artifacts , key = operator .itemgetter ("updated_at" ))
103+ print (
104+ f"Downloading { artifact ['name' ]}
105+ + f"size: { artifact ['size_in_bytes' ]}
106+ + f"created: { utc2local (artifact ['updated_at' ])}
107+ )
108+ download_url (artifact ["archive_download_url" ], temp_zip )
63109 unpack_zipfile (temp_zip )
64110 os .remove (temp_zip )
65- else :
66- print ( f"Fetching artifacts returned status { r . status_code } :" )
67- print ( json . dumps ( r . json (), indent = 4 ))
68- sys .exit (1 )
111+
112+
113+ if __name__ == "__main__" :
114+ sys .exit (main ( * sys . argv [ 1 :]) )
0 commit comments