From 7819799d0d3d16fffba73b203ea600b8fbc64cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Tue, 28 Jan 2025 03:28:11 +0000 Subject: [PATCH] Sort almost all usages of glob.glob for reproducible output This will help diff-ing logs between invocations to see what is changing when refactoring. When used for creating an archive, it will help creating a reproducible file. --- bin/mymd5.py | 2 +- build.py | 82 +++++++++++++++++------------------ buildtools/build_wxwidgets.py | 6 +-- buildtools/config.py | 6 +-- packaging/setup.py | 2 +- setup.py | 2 +- sphinxtools/postprocess.py | 18 ++++---- 7 files changed, 59 insertions(+), 59 deletions(-) diff --git a/bin/mymd5.py b/bin/mymd5.py index a91cc953..b0a170ee 100644 --- a/bin/mymd5.py +++ b/bin/mymd5.py @@ -4,7 +4,7 @@ import hashlib def main(): for arg in sys.argv[1:]: - for name in glob.glob(arg): + for name in sorted(glob.glob(arg)): m = hashlib.md5() with open(name, 'rb') as fid: m.update(fid.read()) diff --git a/build.py b/build.py index 343f1a79..ca27710e 100755 --- a/build.py +++ b/build.py @@ -947,7 +947,7 @@ def do_regenerate_sysconfig(): # grab the file in that folder and copy it into the Python lib p = opj(td, pybd, '*') - datafile = glob.glob(opj(td, pybd, '*'))[0] + datafile = sorted(glob.glob(opj(td, pybd, '*')))[0] cmd = [PYTHON, '-c', 'import sysconfig; print(sysconfig.get_path("stdlib"))'] stdlib = runcmd(cmd, getOutput=True) shutil.copy(datafile, stdlib) @@ -1077,7 +1077,7 @@ def _removeSidebar(path): Remove the sidebar
from the pages going into the docset """ from bs4 import BeautifulSoup - for filename in glob.glob(opj(path, '*.html')): + for filename in sorted(glob.glob(opj(path, '*.html'))): with textfile_open(filename, 'rt') as f: text = f.read() text = text.replace('', '') @@ -1116,7 +1116,7 @@ def cmd_etg(options, args): flags += ' --nodoc' # get the files to run, moving _core the to the front of the list - etgfiles = glob.glob(opj('etg', '_*.py')) + etgfiles = sorted(glob.glob(opj('etg', '_*.py'))) core_file = opj('etg', '_core.py') if core_file in etgfiles: etgfiles.remove(core_file) @@ -1149,13 +1149,13 @@ def cmd_sphinx(options, args): if not os.path.isdir(sphinxDir): raise Exception('Missing sphinx folder in the distribution') - textFiles = glob.glob(sphinxDir + '/*.txt') + textFiles = sorted(glob.glob(sphinxDir + '/*.txt')) if not textFiles: raise Exception('No documentation files found. Please run "build.py touch etg" first') # Copy the rst files into txt files restDir = os.path.join(sphinxDir, 'rest_substitutions', 'overviews') - rstFiles = glob.glob(restDir + '/*.rst') + rstFiles = sorted(glob.glob(restDir + '/*.rst')) for rst in rstFiles: rstName = os.path.split(rst)[1] txt = os.path.join(sphinxDir, os.path.splitext(rstName)[0] + '.txt') @@ -1165,7 +1165,7 @@ def cmd_sphinx(options, args): genGallery() # Copy the hand-edited top level doc files too - rstFiles = glob.glob(os.path.join(phoenixDir(), 'docs', '*.rst')) + rstFiles = sorted(glob.glob(os.path.join(phoenixDir(), 'docs', '*.rst'))) for rst in rstFiles: txt = os.path.join(sphinxDir, os.path.splitext(os.path.basename(rst))[0] + '.txt') copyIfNewer(rst, txt) @@ -1272,7 +1272,7 @@ def cmd_sip(options, args): cmdTimer = CommandTimer('sip') cfg = Config() pwd = pushDir(cfg.ROOT_DIR) - modules = glob.glob(opj(cfg.SIPGEN, '_*.sip')) + modules = sorted(glob.glob(opj(cfg.SIPGEN, '_*.sip'))) # move _core the to the front of the list core_file = opj(cfg.SIPGEN, '_core.sip') if core_file in modules: @@ -1393,7 +1393,7 @@ def cmd_sip(options, args): # Check each file in tmpdir to see if it is different than the same file # in cfg.SIPOUT. If so then copy the new one to cfg.SIPOUT, otherwise # ignore it. - for src in glob.glob(sip_tmp_out_dir + '/*'): + for src in sorted(glob.glob(sip_tmp_out_dir + '/*')): dest = opj(cfg.SIPOUT, os.path.basename(src)) if not os.path.exists(dest): msg('%s is a new file, copying...' % os.path.basename(src)) @@ -1421,7 +1421,7 @@ def cmd_sip(options, args): with tempfile.TemporaryDirectory() as tmpdir: cmd = 'sip-module --sdist --abi-version {} --target-dir {} wx.siplib'.format(cfg.SIP_ABI, tmpdir) runcmd(cmd) - tf_name = glob.glob(tmpdir + '/*.tar*')[0] + tf_name = sorted(glob.glob(tmpdir + '/*.tar*'))[0] tf_dir = os.path.splitext(os.path.splitext(tf_name)[0])[0] with tarfile.open(tf_name) as tf: try: @@ -1439,7 +1439,7 @@ def cmd_touch(options, args): cmdTimer = CommandTimer('touch') pwd = pushDir(phoenixDir()) etg = pathlib.Path('etg') - for item in etg.glob('*.py'): + for item in sorted(etg.glob('*.py')): item.touch() cmd_touch_others(options, args) @@ -1631,21 +1631,21 @@ def copyWxDlls(options): arch = 'x64' if PYTHON_ARCH == '64bit' else 'x86' dlls = list() if not options.debug or options.both: - dlls += glob.glob(os.path.join(msw.dllDir, "wx*%su_*.dll" % ver)) + dlls += sorted(glob.glob(os.path.join(msw.dllDir, "wx*%su_*.dll" % ver))) if options.relwithdebug: - dlls += glob.glob(os.path.join(msw.dllDir, "wx*%su_*.pdb" % ver)) + dlls += sorted(glob.glob(os.path.join(msw.dllDir, "wx*%su_*.pdb" % ver))) if options.debug or options.both: - dlls += glob.glob(os.path.join(msw.dllDir, "wx*%sud_*.dll" % ver)) - dlls += glob.glob(os.path.join(msw.dllDir, "wx*%sud_*.pdb" % ver)) + dlls += sorted(glob.glob(os.path.join(msw.dllDir, "wx*%sud_*.dll" % ver))) + dlls += sorted(glob.glob(os.path.join(msw.dllDir, "wx*%sud_*.pdb" % ver))) # Also copy the cairo DLLs if needed if options.cairo: cairo_root = os.path.join(phoenixDir(), 'packaging', 'msw-cairo') - dlls += glob.glob(os.path.join(cairo_root, arch, 'bin', '*.dll')) + dlls += sorted(glob.glob(os.path.join(cairo_root, arch, 'bin', '*.dll'))) # And the webview2 (MS EDGE) DLL wv2_root = os.path.join(phoenixDir(), 'packaging', 'msw-webview2') - dlls += glob.glob(os.path.join(wv2_root, arch, '*.dll')) + dlls += sorted(glob.glob(os.path.join(wv2_root, arch, '*.dll'))) # For Python 3.5 and 3.6 builds we also need to copy some VC14 redist DLLs. # NOTE: Do it for 3.7+ too for now. But when we fully switch over to VS 2017 @@ -1654,7 +1654,7 @@ def copyWxDlls(options): redist_dir = os.path.join( phoenixDir(), 'packaging', 'msw-vcredist', arch, 'Microsoft.VC140.CRT', '*.dll') - dlls += glob.glob(redist_dir) + dlls += sorted(glob.glob(redist_dir)) for dll in dlls: copyIfNewer(dll, posixjoin(phoenixDir(), cfg.PKGDIR, os.path.basename(dll)), verbose=True) @@ -1663,22 +1663,22 @@ def copyWxDlls(options): # Copy the wxWidgets dylibs cfg = Config() wxlibdir = os.path.join(getBuildDir(options), "lib") - dlls = glob.glob(wxlibdir + '/*.dylib') + dlls = sorted(glob.glob(wxlibdir + '/*.dylib')) for dll in dlls: copyIfNewer(dll, posixjoin(phoenixDir(), cfg.PKGDIR, os.path.basename(dll)), verbose=True) # Now use install_name_tool to change the extension modules to look # in the same folder for the wx libs, instead of the build dir. Also # change the wx libs the same way. - macSetLoaderNames(glob.glob(opj(phoenixDir(), cfg.PKGDIR, '*.so')) + - glob.glob(opj(phoenixDir(), cfg.PKGDIR, '*.dylib'))) + macSetLoaderNames(sorted(glob.glob(opj(phoenixDir(), cfg.PKGDIR, '*.so'))) + + sorted(glob.glob(opj(phoenixDir(), cfg.PKGDIR, '*.dylib')))) else: # Not Windows and not OSX. For now that means that we'll assume it's wxGTK. cfg = Config() wxlibdir = os.path.join(getBuildDir(options), "lib") - dlls = glob.glob(wxlibdir + '/libwx_*.so') - dlls += glob.glob(wxlibdir + '/libwx_*.so.[0-9]*') + dlls = sorted(glob.glob(wxlibdir + '/libwx_*.so')) + dlls += sorted(glob.glob(wxlibdir + '/libwx_*.so.[0-9]*')) for dll in dlls: copyIfNewer(dll, posixjoin(phoenixDir(), cfg.PKGDIR, os.path.basename(dll)), verbose=True) @@ -1872,7 +1872,7 @@ def cmd_touch_others(options, args): pwd = pushDir(phoenixDir()) cfg = Config(noWxConfig=True) pth = pathlib.Path(opj(cfg.PKGDIR, 'svg')) - for item in pth.glob('*.pyx'): + for item in sorted(pth.glob('*.pyx')): item.touch() @@ -1882,7 +1882,7 @@ def cmd_clean_others(options, args): cfg = Config(noWxConfig=True) files = [] for wc in ['*.pyd', '*.so']: - files += glob.glob(opj(cfg.PKGDIR, 'svg', wc)) + files += sorted(glob.glob(opj(cfg.PKGDIR, 'svg', wc))) delFiles(files) @@ -1931,7 +1931,7 @@ def cmd_build_pdbzip(options, args): os.mkdir('dist') cfg = Config() - filenames = glob.glob('./wx/*.pdb') + filenames = sorted(glob.glob('./wx/*.pdb')) if not filenames: msg('No PDB files found in ./wx!') return @@ -1963,7 +1963,7 @@ def cmd_bdist_egg(options, args): cfg = Config() if options.upload: filemask = "dist/%s-%s-*.egg" % (baseName, cfg.VERSION) - filenames = glob.glob(filemask) + filenames = sorted(glob.glob(filemask)) assert len(filenames) == 1, "Unknown files found:"+repr(filenames) uploadPackage(filenames[0], options) if pdbzip: @@ -1976,10 +1976,10 @@ def cmd_bdist_wheel(options, args): cfg = Config() if options.upload: filemask = "dist/%s-%s-*.whl" % (baseName, cfg.VERSION) - filenames = glob.glob(filemask) + filenames = sorted(glob.glob(filemask)) print(f'**** filemask: {filemask}') print(f'**** matched: {filenames}') - print(f'**** all dist: {glob.glob("dist/*")}') + print(f'**** all dist: {sorted(glob.glob("dist/*"))}') assert len(filenames) == 1, "Unknown files found:"+repr(filenames) uploadPackage(filenames[0], options) @@ -1994,7 +1994,7 @@ def cmd_bdist_wininst(options, args): cfg = Config() if options.upload: filemask = "dist/%s-%s-*.exe" % (baseName, cfg.VERSION) - filenames = glob.glob(filemask) + filenames = sorted(glob.glob(filemask)) assert len(filenames) == 1, "Unknown files found:"+repr(filenames) uploadPackage(filenames[0], options) if pdbzip: @@ -2022,8 +2022,8 @@ def cmd_clean_wx(options, args): options.debug = True msw = getMSWSettings(options) deleteIfExists(opj(msw.dllDir, 'msw'+msw.dll_type)) - delFiles(glob.glob(opj(msw.dllDir, 'wx*%s%s*' % (wxversion2_nodot, msw.dll_type)))) - delFiles(glob.glob(opj(msw.dllDir, 'wx*%s%s*' % (wxversion3_nodot, msw.dll_type)))) + delFiles(sorted(glob.glob(opj(msw.dllDir, 'wx*%s%s*' % (wxversion2_nodot, msw.dll_type))))) + delFiles(sorted(glob.glob(opj(msw.dllDir, 'wx*%s%s*' % (wxversion3_nodot, msw.dll_type))))) if PYTHON_ARCH == '64bit': deleteIfExists(opj(msw.buildDir, 'vc%s_x64_msw%sdll' % (getVisCVersion(), msw.dll_type))) else: @@ -2048,19 +2048,19 @@ def cmd_clean_py(options, args): deleteIfExists(getWafBuildBase()) files = list() for wc in ['*.py', '*.pyc', '*.so', '*.dylib', '*.pyd', '*.pdb', '*.pi', '*.pyi']: - files += glob.glob(opj(cfg.PKGDIR, wc)) + files += sorted(glob.glob(opj(cfg.PKGDIR, wc))) if isWindows: msw = getMSWSettings(options) for wc in [ 'wx*' + wxversion2_nodot + msw.dll_type + '*.dll', 'wx*' + wxversion3_nodot + msw.dll_type + '*.dll']: - files += glob.glob(opj(cfg.PKGDIR, wc)) + files += sorted(glob.glob(opj(cfg.PKGDIR, wc))) delFiles(files) # Also remove any remaining DLLs just to make sure. This includes the C++ # runtime DLLs, Cairo, etc. # TODO: Check for specific files, not just *.dll if isWindows: - files = glob.glob(opj(cfg.PKGDIR, '*.dll')) + files = sorted(glob.glob(opj(cfg.PKGDIR, '*.dll'))) delFiles(files) @@ -2090,7 +2090,7 @@ def cmd_clean_sphinx(options, args): opj(sphinxDir, '_static/images/inheritance/*.*'), ] for wc in globs: - for f in glob.glob(wc): + for f in sorted(glob.glob(wc)): os.remove(f) dirs = [opj(sphinxDir, 'build'), @@ -2132,7 +2132,7 @@ def cmd_cleanall(options, args): assert os.getcwd() == phoenixDir() files = list() for wc in ['sip/cpp/*.h', 'sip/cpp/*.cpp', 'sip/cpp/*.sbf', 'sip/gen/*.sip']: - files += glob.glob(wc) + files += sorted(glob.glob(wc)) delFiles(files) cmd_clean_docker(options, args) @@ -2204,14 +2204,14 @@ def cmd_sdist(options, args): os.makedirs(posixjoin(PDEST, 'sip', 'siplib'), exist_ok=True) for srcdir in ['cpp', 'gen', 'siplib']: destdir = posixjoin(PDEST, 'sip', srcdir) - for name in glob.glob(posixjoin('sip', srcdir, '*')): + for name in sorted(glob.glob(posixjoin('sip', srcdir, '*'))): if not os.path.isdir(name): copyFile(name, destdir) sip_h_dir = posixjoin(cfg.PKGDIR, 'include', 'wxPython') copyFile(posixjoin(sip_h_dir, 'sip.h'), posixjoin(PDEST, sip_h_dir)) for wc in ['*.py', '*.pi', '*.pyi']: destdir = posixjoin(PDEST, cfg.PKGDIR) - for name in glob.glob(posixjoin(cfg.PKGDIR, wc)): + for name in sorted(glob.glob(posixjoin(cfg.PKGDIR, wc))): copyFile(name, destdir) copyFile('demo/version.py', posixjoin(PDEST, 'demo')) @@ -2260,7 +2260,7 @@ def cmd_sdist(options, args): os.remove(tarfilename) pwd = pushDir(PDEST) with tarfile.open(name=tarfilename, mode="w:gz") as tarball: - for name in glob.glob('*'): + for name in sorted(glob.glob('*')): tarball.add(name, os.path.join(rootname, name), filter=_setTarItemPerms) msg('Cleaning up...') @@ -2314,7 +2314,7 @@ def cmd_sdist_demo(options, args): os.remove(tarfilename) pwd = pushDir(PDEST) with tarfile.open(name=tarfilename, mode="w:gz") as tarball: - for name in glob.glob('*'): + for name in sorted(glob.glob('*')): tarball.add(name, os.path.join(rootname, name), filter=_setTarItemPerms) msg('Cleaning up...') @@ -2366,7 +2366,7 @@ def cmd_bdist(options, args): # If the DLLs are not already in the wx package folder then go fetch # them now. msg("Archiving wxWidgets shared libraries...") - dlls = glob.glob(os.path.join(wxlibdir, "*%s" % dllext)) + dlls = sorted(glob.glob(os.path.join(wxlibdir, "*%s" % dllext))) for dll in dlls: tarball.add(dll, os.path.join(rootname, 'wx', os.path.basename(dll))) diff --git a/buildtools/build_wxwidgets.py b/buildtools/build_wxwidgets.py index 973dbc46..5c3a4a6c 100644 --- a/buildtools/build_wxwidgets.py +++ b/buildtools/build_wxwidgets.py @@ -117,7 +117,7 @@ def macFixupInstallNames(destdir, prefix, buildDir=None): print("**** macFixupInstallNames(%s, %s, %s)" % (destdir, prefix, buildDir)) pwd = os.getcwd() os.chdir(destdir+prefix+'/lib') - dylibs = glob.glob('*.dylib') # ('*[0-9].[0-9].[0-9].[0-9]*.dylib') + dylibs = sorted(glob.glob('*.dylib')) # ('*[0-9].[0-9].[0-9].[0-9]*.dylib') for lib in dylibs: cmd = 'install_name_tool -id %s/lib/%s %s/lib/%s' % \ (prefix,lib, destdir+prefix,lib) @@ -571,7 +571,7 @@ def main(wxDir, args): renameLibrary(libfile, "wx" + lib) run("ln -s -f ../../../%s %s/wx%s" % (libfile, frameworkDir, lib)) - for lib in glob.glob("lib/*.dylib"): + for lib in sorted(glob.glob("lib/*.dylib")): if not os.path.islink(lib): corelibname = "lib/lib%s-%s.0.dylib" % (basename, version) run("install_name_tool -id %s %s" % (os.path.join(prefixDir, lib), lib)) @@ -589,7 +589,7 @@ def main(wxDir, args): """ headers = "" header_dir = "wx-%s/wx" % version - for include in glob.glob(header_dir + "/*.h"): + for include in sorted(glob.glob(header_dir + "/*.h")): headers += "#include \n" with open("%s.h" % fwname, "w") as framework_header: diff --git a/buildtools/config.py b/buildtools/config.py index cf7635ae..dda3c675 100644 --- a/buildtools/config.py +++ b/buildtools/config.py @@ -472,7 +472,7 @@ class Configuration(object): def build_locale_dir(self, destdir, verbose=1): """Build a locale dir under the wxPython package.""" - moFiles = glob.glob(opj(self.WXDIR, 'locale', '*.mo')) + moFiles = sorted(glob.glob(opj(self.WXDIR, 'locale', '*.mo'))) for src in moFiles: lang = os.path.splitext(os.path.basename(src))[0] dest = opj(destdir, lang, 'LC_MESSAGES') @@ -525,7 +525,7 @@ class Configuration(object): else: walk_helper((file_list, wildcards), srcdir, - [os.path.basename(f) for f in glob.glob(opj(srcdir, '*'))]) + [os.path.basename(f) for f in sorted(glob.glob(opj(srcdir, '*')))]) return file_list @@ -818,7 +818,7 @@ def macFixDependencyInstallName(destdir, prefix, extension, buildDir): print("**** macFixDependencyInstallName(%s, %s, %s, %s)" % (destdir, prefix, extension, buildDir)) pwd = os.getcwd() os.chdir(destdir+prefix+'/lib') - dylibs = glob.glob('*.dylib') + dylibs = sorted(glob.glob('*.dylib')) for lib in dylibs: #cmd = 'install_name_tool -change %s/lib/%s %s/lib/%s %s' % \ # (destdir+prefix,lib, prefix,lib, extension) diff --git a/packaging/setup.py b/packaging/setup.py index 6df4be60..ef1b6c7b 100644 --- a/packaging/setup.py +++ b/packaging/setup.py @@ -33,7 +33,7 @@ if os.path.exists(SRC) and os.path.isdir(SRC): for wc in ['wxWidgets/configure', 'wxWidgets/src/stc/gen_iface.py', 'bin/waf-*', ]: - for item in glob.glob(wc): + for item in sorted(glob.glob(wc)): os.chmod(item, 0o755) diff --git a/setup.py b/setup.py index 99ee6612..62704bfd 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def _cleanup_symlinks(cmd): # build_lib = cmd.get_finalized_command('build').build_lib build_lib = opj(build_lib, 'wx') - for libname in glob.glob(opj(build_lib, 'libwx*')): + for libname in sorted(glob.glob(opj(build_lib, 'libwx*'))): if os.path.islink(libname): if isDarwin: diff --git a/sphinxtools/postprocess.py b/sphinxtools/postprocess.py index e6597025..65d82512 100644 --- a/sphinxtools/postprocess.py +++ b/sphinxtools/postprocess.py @@ -76,7 +76,7 @@ def genIndexes(sphinxDir): files. """ print("Generating indexes...") - pklfiles = glob.glob(sphinxDir + '/*.pkl') + pklfiles = sorted(glob.glob(sphinxDir + '/*.pkl')) for file in pklfiles: if file.endswith('functions.pkl'): @@ -109,8 +109,8 @@ def buildEnumsAndMethods(sphinxDir): unreferenced_classes = {} - textfiles = glob.glob(sphinxDir + '/*.txt') - enum_files = glob.glob(sphinxDir + '/*.enumeration.txt') + textfiles = sorted(glob.glob(sphinxDir + '/*.txt')) + enum_files = sorted(glob.glob(sphinxDir + '/*.enumeration.txt')) enum_base = [os.path.split(os.path.splitext(enum)[0])[1] for enum in enum_files] enum_base = [enum.replace('.enumeration', '') for enum in enum_base] @@ -426,12 +426,12 @@ def makeModuleIndex(sphinxDir, file): # "wx.lower.UpperName". This is so we don't put all the enums in the # submodules in the core wx module too. # TODO: This may not work on case-insensitive file systems, check it. - enum_files = glob.glob(sphinxDir + '/wx.[A-Z]*.enumeration.txt') + enum_files = sorted(glob.glob(sphinxDir + '/wx.[A-Z]*.enumeration.txt')) else: label = '.'.join(local_file.split('.')[0:2]) module = label enumDots = 3 - enum_files = glob.glob(sphinxDir + '/%s*.enumeration.txt' % module) + enum_files = sorted(glob.glob(sphinxDir + '/%s*.enumeration.txt' % module)) enum_base = [os.path.split(os.path.splitext(enum)[0])[1] for enum in enum_files] @@ -531,11 +531,11 @@ def genGallery(): plat_folder = os.path.join(image_folder, folder) os.chdir(plat_folder) - image_files[folder] = glob.glob('*.png') + image_files[folder] = sorted(glob.glob('*.png')) os.chdir(pwd) - txt_files = glob.glob(SPHINXROOT + '/*.txt') + txt_files = sorted(glob.glob(SPHINXROOT + '/*.txt')) html_files = {} for text in txt_files: @@ -647,9 +647,9 @@ def addJavaScript(text): # ----------------------------------------------------------------------- # def postProcess(folder, options): - fileNames = glob.glob(folder + "/*.html") + fileNames = sorted(glob.glob(folder + "/*.html")) - enum_files = glob.glob(folder + '/*.enumeration.html') + enum_files = sorted(glob.glob(folder + '/*.enumeration.html')) enum_base = [os.path.split(os.path.splitext(enum)[0])[1] for enum in enum_files] enum_base = [enum.replace('.enumeration', '') for enum in enum_base]