- tbb-commits - lists.torproject.org

[tor-browser-build/master] Bug 40252: Bump mingw-w64 and clang for Firefox 78.9
by sysrqb＠torproject.org 22 Mar '21

22 Mar '21

commit bd6a4ce633d0c4e09690f7113bd19593a9188ff7 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Mar 18 11:09:59 2021 +0100 Bug 40252: Bump mingw-w64 and clang for Firefox 78.9 https://hg.mozilla.org/releases/mozilla-esr78/rev/1b0e1f0d885e https://hg.mozilla.org/releases/mozilla-esr78/rev/d726d74daa39 https://hg.mozilla.org/releases/mozilla-esr78/rev/cc840a008393 --- projects/binutils/config | 5 + projects/clang-source/build | 15 +-- projects/clang-source/config | 9 +- projects/clang-source/no-seh.patch | 145 -------------------------- projects/clang/build | 2 + projects/clang/config | 8 ++ projects/mingw-w64-clang/build | 4 + projects/mingw-w64-clang/config | 3 +- projects/mingw-w64-clang/mingw-dwrite_3.patch | 84 +++++++++++++++ projects/mingw-w64/config | 2 +- 10 files changed, 118 insertions(+), 159 deletions(-) diff --git a/projects/binutils/config b/projects/binutils/config index 48065e1..9fe4496 100644 --- a/projects/binutils/config +++ b/projects/binutils/config @@ -7,6 +7,11 @@ var: use_container: 1 targets: + # We use the windows-clang target for the binutils we use for building + # clang during windows builds. See projects/clang/config. + windows-clang: + var: + configure_opt: '--disable-multilib --enable-gold --enable-deterministic-archives --enable-plugins' windows: var: configure_opt: '--target=[% c("arch") %]-w64-mingw32 --disable-multilib --enable-deterministic-archives' diff --git a/projects/clang-source/build b/projects/clang-source/build index d94fcbe..4f824a6 100644 --- a/projects/clang-source/build +++ b/projects/clang-source/build @@ -10,22 +10,17 @@ tar -xf $rootdir/[% c('input_files_by_name/compiler-rt') %] tar -xf $rootdir/[% c('input_files_by_name/libunwind') %] mv llvm-[% c("version") %].src llvm -# LLVM has reproducibility issues when optimizing bitcode, which we need to -# patch. See: #32053 for more details. -patch -p1 < $rootdir/43909.patch +[% IF ! c("var/windows") -%] + # LLVM has reproducibility issues when optimizing bitcode, which we need to + # patch. See: #32053 for more details. + patch -p1 < $rootdir/43909.patch +[% END -%] mv clang-[% c("version") %].src llvm/tools/clang # Having clang-tidy available seems like a good idea mv clang-tools-extra-[% c("version") %].src llvm/tools/clang/tools/extra mv libcxx-[% c("version") %].src llvm/projects/libcxx mv libcxxabi-[% c("version") %].src llvm/projects/libcxxabi mv lld-[% c("version") %].src llvm/tools/lld -[% IF c("var/windows-i686") %] - # mingw-w64 does not support SEH on 32bit systems. Make it possible to - # explicitly disable it. - cd llvm/tools - patch -p1 < $rootdir/no-seh.patch - cd ../../ -[% END %] mv compiler-rt-[% c("version") %].src llvm/projects/compiler-rt # We need libunwind only for mingw-w64-clang, don't include it as tightly by # putting it into projects/ diff --git a/projects/clang-source/config b/projects/clang-source/config index 8adb18e..a6dce85 100644 --- a/projects/clang-source/config +++ b/projects/clang-source/config @@ -4,6 +4,12 @@ filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' gpg_keyring: clang.gpg sig_ext: sig +targets: + windows: + # We use clang 11 for Windows builds: + # https://hg.mozilla.org/releases/mozilla-esr78/rev/cc840a008393 + version: 11.0.1 + input_files: - URL: 'https://github.com/llvm/llvm-project/releases/download/llvmorg-[% c("version") %]/llvm-[% c("version") %].src.tar.xz' name: llvm @@ -30,5 +36,4 @@ input_files: name: libunwind file_gpg_id: 1 - filename: 43909.patch - - filename: no-seh.patch - enable: '[% c("var/windows-i686") %]' + enable: '[% ! c("var/windows") %]' diff --git a/projects/clang-source/no-seh.patch b/projects/clang-source/no-seh.patch deleted file mode 100644 index 1a52e40..0000000 --- a/projects/clang-source/no-seh.patch +++ /dev/null @@ -1,145 +0,0 @@ -From 38cbe873d45cf3c881ef4113b48193edfd418f43 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin(a)martin.st> -Date: Mon, 27 Jul 2020 23:44:41 +0300 -Subject: [PATCH] Implement the --no-seh flag - -Previously this flag was just ignored. If set, set the -IMAGE_DLL_CHARACTERISTICS_NO_SEH bit, regardless of the normal safeSEH -machinery. - -In mingw configurations, the safeSEH bit might not be set in e.g. object -files built from handwritten assembly, making it impossible to use the -normal safeseh flag. As mingw setups don't generally use SEH on 32 bit -x86 at all, it should be fine to set that flag bit though - hook up -the existing GNU ld flag for controlling that. - -Differential Revision: https://reviews.llvm.org/D84701 - -diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h -index 4b62cd05f4f..17ea0f8bcab 100644 ---- a/lld/COFF/Config.h -+++ b/lld/COFF/Config.h -@@ -135,6 +135,7 @@ struct Configuration { - bool safeSEH = false; - Symbol *sehTable = nullptr; - Symbol *sehCount = nullptr; -+ bool noSEH = false; - - // Used for /opt:lldlto=N - unsigned ltoo = 2; -diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp -index 15d6fb5121a..b569df07601 100644 ---- a/lld/COFF/Driver.cpp -+++ b/lld/COFF/Driver.cpp -@@ -1574,9 +1574,10 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) { - config->wordsize = config->is64() ? 8 : 4; - - // Handle /safeseh, x86 only, on by default, except for mingw. -- if (config->machine == I386 && -- args.hasFlag(OPT_safeseh, OPT_safeseh_no, !config->mingw)) -- config->safeSEH = true; -+ if (config->machine == I386) { -+ config->safeSEH = args.hasFlag(OPT_safeseh, OPT_safeseh_no, !config->mingw); -+ config->noSEH = args.hasArg(OPT_noseh); -+ } - - // Handle /functionpadmin - for (auto *arg : args.filtered(OPT_functionpadmin, OPT_functionpadmin_opt)) -diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td -index 024b7be8f78..c7e2a5ea57e 100644 ---- a/lld/COFF/Options.td -+++ b/lld/COFF/Options.td -@@ -171,6 +171,7 @@ def include_optional : Joined<["/", "-", "/?", "-?"], "includeoptional:">, - HelpText<"Add symbol as undefined, but allow it to remain undefined">; - def kill_at : F<"kill-at">; - def lldmingw : F<"lldmingw">; -+def noseh : F<"noseh">; - def output_def : Joined<["/", "-", "/?", "-?"], "output-def:">; - def pdb_source_path : P<"pdbsourcepath", - "Base path used to make relative source file path absolute in PDB">; -diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp -index 5736281958f..aead781d38b 100644 ---- a/lld/COFF/Writer.cpp -+++ b/lld/COFF/Writer.cpp -@@ -1352,7 +1352,7 @@ template <typename PEHeaderTy> void Writer::writeHeader() { - pe->DLLCharacteristics |= IMAGE_DLL_CHARACTERISTICS_GUARD_CF; - if (config->integrityCheck) - pe->DLLCharacteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; -- if (setNoSEHCharacteristic) -+ if (setNoSEHCharacteristic || config->noSEH) - pe->DLLCharacteristics |= IMAGE_DLL_CHARACTERISTICS_NO_SEH; - if (config->terminalServerAware) - pe->DLLCharacteristics |= IMAGE_DLL_CHARACTERISTICS_TERMINAL_SERVER_AWARE; -diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp -index be1b757e45b..1d292817cfb 100644 ---- a/lld/MinGW/Driver.cpp -+++ b/lld/MinGW/Driver.cpp -@@ -249,6 +249,8 @@ bool mingw::link(ArrayRef<const char *> argsArr, raw_ostream &diag) { - add("-kill-at"); - if (args.hasArg(OPT_appcontainer)) - add("-appcontainer"); -+ if (args.hasArg(OPT_no_seh)) -+ add("-noseh"); - - if (args.getLastArgValue(OPT_m) != "thumb2pe" && - args.getLastArgValue(OPT_m) != "arm64pe" && !args.hasArg(OPT_dynamicbase)) -diff --git a/lld/MinGW/Options.td b/lld/MinGW/Options.td -index 86400433d04..931cf264837 100644 ---- a/lld/MinGW/Options.td -+++ b/lld/MinGW/Options.td -@@ -45,6 +45,7 @@ def minor_subsystem_version_eq: Joined<["--"], "minor-subsystem-version=">, - Alias<minor_subsystem_version>; - def no_insert_timestamp: F<"no-insert-timestamp">, - HelpText<"Don't include PE header timestamp">; -+def no_seh: F<"no-seh">, HelpText<"Set the 'no SEH' flag in the executable">; - def no_whole_archive: F<"no-whole-archive">, - HelpText<"No longer include all object files for following archives">; - def large_address_aware: Flag<["--"], "large-address-aware">, -@@ -104,7 +105,6 @@ def: Flag<["--"], "full-shutdown">; - def: F<"high-entropy-va">; - def: S<"major-image-version">; - def: S<"minor-image-version">; --def: F<"no-seh">; - def: F<"nxcompat">; - def: F<"pic-executable">; - def: S<"plugin">; -diff --git a/lld/test/COFF/noseh.s b/lld/test/COFF/noseh.s -new file mode 100644 -index 00000000000..44295228622 ---- /dev/null -+++ b/lld/test/COFF/noseh.s -@@ -0,0 +1,19 @@ -+# REQUIRES: x86 -+# RUN: llvm-mc -triple i686-w64-mingw32 %s -filetype=obj -o %t.obj -+# RUN: lld-link -lldmingw %t.obj -out:%t.exe -entry:main -+# RUN: llvm-readobj --file-headers %t.exe | FileCheck %s --check-prefix=DEFAULT -+# RUN: lld-link -lldmingw %t.obj -out:%t.noseh.exe -entry:main -noseh -+# RUN: llvm-readobj --file-headers %t.noseh.exe | FileCheck %s --check-prefix=NOSEH -+ -+# DEFAULT: Characteristics [ -+# DEFAULT-NOT: IMAGE_DLL_CHARACTERISTICS_NO_SEH -+# DEFAULT: ] -+ -+# NOSEH: Characteristics [ -+# NOSEH: IMAGE_DLL_CHARACTERISTICS_NO_SEH -+# NOSEH: ] -+ -+ .text -+ .globl _main -+_main: -+ ret -diff --git a/lld/test/MinGW/driver.test b/lld/test/MinGW/driver.test -index b8bc2ddea8a..1d53205927b 100644 ---- a/lld/test/MinGW/driver.test -+++ b/lld/test/MinGW/driver.test -@@ -204,3 +204,7 @@ APPCONTAINER: -appcontainer - RUN: ld.lld -### -m i386pep foo.o -delayload user32.dll --delayload shell32.dll | FileCheck -check-prefix DELAYLOAD %s - RUN: ld.lld -### -m i386pep foo.o -delayload=user32.dll --delayload=shell32.dll | FileCheck -check-prefix DELAYLOAD %s - DELAYLOAD: -delayload:user32.dll -delayload:shell32.dll -+ -+RUN: ld.lld -### -m i386pe foo.o -no-seh | FileCheck -check-prefix NOSEH %s -+RUN: ld.lld -### -m i386pe foo.o --no-seh | FileCheck -check-prefix NOSEH %s -+NOSEH: -noseh --- -2.28.0 - diff --git a/projects/clang/build b/projects/clang/build index c4c16d9..05a93dc 100644 --- a/projects/clang/build +++ b/projects/clang/build @@ -12,6 +12,8 @@ export PATH="/var/tmp/dist/cmake/bin:$PATH" hardened_gcc => 0 }) %] ln -s gcc /var/tmp/dist/gcc/bin/cc [% END %] +[% END -%] +[% IF c("var/linux") || c("var/android") || c("var/windows") -%] tar -C /var/tmp/dist -xf [% c('input_files_by_name/binutils') %] export PATH="/var/tmp/dist/binutils/bin:$PATH" [% END -%] diff --git a/projects/clang/config b/projects/clang/config index 6038eab..2d12335 100644 --- a/projects/clang/config +++ b/projects/clang/config @@ -20,6 +20,14 @@ input_files: - name: binutils project: binutils enable: '[% c("var/linux") %] || [% c("var/android") %]' + # We build our own binutils as the one from buster has a regression: + # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=953423 + # https://sourceware.org/bugzilla/show_bug.cgi?id=24458 + - name: binutils + project: binutils + enable: '[% c("var/windows") %]' + target_prepend: + - windows-clang - project: cmake name: cmake - project: clang-source diff --git a/projects/mingw-w64-clang/build b/projects/mingw-w64-clang/build index 44a913f..974d2a9 100644 --- a/projects/mingw-w64-clang/build +++ b/projects/mingw-w64-clang/build @@ -60,6 +60,7 @@ default_win32_winnt=0x601 cd $builddir/mingw-w64-clang patch -p1 < $rootdir/mingw-winrt.patch +patch -p1 < $rootdir/mingw-dwrite_3.patch cd mingw-w64-headers mkdir build && cd build @@ -164,6 +165,9 @@ cmake -DCMAKE_BUILD_TYPE=Release \ make -j[% c("buildconf/num_procs") %] VERBOSE=1 # libcxx +# the build expects ../llvm to exist so we add a symlink +# See https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/merge_re… +ln -s /var/tmp/build/clang-source /var/tmp/build/clang-source/projects/llvm cd $builddir/clang-source/projects/libcxx mkdir build && cd build cmake -DCMAKE_BUILD_TYPE=Release \ diff --git a/projects/mingw-w64-clang/config b/projects/mingw-w64-clang/config index 1bf5954..080c75b 100644 --- a/projects/mingw-w64-clang/config +++ b/projects/mingw-w64-clang/config @@ -1,7 +1,7 @@ # vim: filetype=yaml sw=2 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' git_url: https://git.code.sf.net/p/mingw-w64/mingw-w64 -git_hash: 1b373beec6d07478ffba33726bb3bb21f32e4411 +git_hash: 660e09f3cb20f181b6d6435cb623d65a3922a063 version: '[% c("abbrev") %]' var: @@ -23,3 +23,4 @@ input_files: - project: llvm-mingw name: llvm-mingw - filename: mingw-winrt.patch + - filename: mingw-dwrite_3.patch diff --git a/projects/mingw-w64-clang/mingw-dwrite_3.patch b/projects/mingw-w64-clang/mingw-dwrite_3.patch new file mode 100644 index 0000000..7c7b8c9 --- /dev/null +++ b/projects/mingw-w64-clang/mingw-dwrite_3.patch @@ -0,0 +1,84 @@ +From 86f4e9cbb5d3099179dc51332970d61d394116f6 Mon Sep 17 00:00:00 2001 +From: Sanketh Menda <me(a)snkth.com> +Date: Sat, 17 Oct 2020 15:39:42 -0400 +Subject: [PATCH] dwrite_3.h: rename GetGlyphImageFormats_ to + GetGlyphImageFormats + +Wine's WIDL currently doesn't support overloading functions, so till +that is fixed patch dwrite_3.h to rename GlyphImageFormats_ to +GetGlyphImageFormats. +--- + mingw-w64-headers/include/dwrite_3.h | 18 +++++++++--------- + 1 file changed, 9 insertions(+), 9 deletions(-) + +diff --git a/mingw-w64-headers/include/dwrite_3.h b/mingw-w64-headers/include/dwrite_3.h +index 427c39e3b..dd894a8b1 100644 +--- a/mingw-w64-headers/include/dwrite_3.h ++++ b/mingw-w64-headers/include/dwrite_3.h +@@ -8168,7 +8168,7 @@ DEFINE_GUID(IID_IDWriteFontFace4, 0x27f2a904, 0x4eb8, 0x441d, 0x96,0x78, 0x05,0x + MIDL_INTERFACE("27f2a904-4eb8-441d-9678-0563f53e3e2f") + IDWriteFontFace4 : public IDWriteFontFace3 + { +- virtual HRESULT STDMETHODCALLTYPE GetGlyphImageFormats_( ++ virtual HRESULT STDMETHODCALLTYPE GetGlyphImageFormats( + UINT16 glyph, + UINT32 ppem_first, + UINT32 ppem_last, +@@ -8468,7 +8468,7 @@ typedef struct IDWriteFontFace4Vtbl { + WINBOOL *are_local); + + /*** IDWriteFontFace4 methods ***/ +- HRESULT (STDMETHODCALLTYPE *GetGlyphImageFormats_)( ++ HRESULT (STDMETHODCALLTYPE *GetGlyphImageFormats)( + IDWriteFontFace4 *This, + UINT16 glyph, + UINT32 ppem_first, +@@ -8549,7 +8549,7 @@ interface IDWriteFontFace4 { + #define IDWriteFontFace4_AreCharactersLocal(This,characters,count,enqueue_if_not,are_local) (This)->lpVtbl->AreCharactersLocal(This,characters,count,enqueue_if_not,are_local) + #define IDWriteFontFace4_AreGlyphsLocal(This,glyphs,count,enqueue_if_not,are_local) (This)->lpVtbl->AreGlyphsLocal(This,glyphs,count,enqueue_if_not,are_local) + /*** IDWriteFontFace4 methods ***/ +-#define IDWriteFontFace4_GetGlyphImageFormats_(This,glyph,ppem_first,ppem_last,formats) (This)->lpVtbl->GetGlyphImageFormats_(This,glyph,ppem_first,ppem_last,formats) ++#define IDWriteFontFace4_GetGlyphImageFormats(This,glyph,ppem_first,ppem_last,formats) (This)->lpVtbl->GetGlyphImageFormats(This,glyph,ppem_first,ppem_last,formats) + #define IDWriteFontFace4_GetGlyphImageFormats(This) (This)->lpVtbl->GetGlyphImageFormats(This) + #define IDWriteFontFace4_GetGlyphImageData(This,glyph,ppem,format,data,context) (This)->lpVtbl->GetGlyphImageData(This,glyph,ppem,format,data,context) + #define IDWriteFontFace4_ReleaseGlyphImageData(This,context) (This)->lpVtbl->ReleaseGlyphImageData(This,context) +@@ -8692,8 +8692,8 @@ static FORCEINLINE HRESULT IDWriteFontFace4_AreGlyphsLocal(IDWriteFontFace4* Thi + return This->lpVtbl->AreGlyphsLocal(This,glyphs,count,enqueue_if_not,are_local); + } + /*** IDWriteFontFace4 methods ***/ +-static FORCEINLINE HRESULT IDWriteFontFace4_GetGlyphImageFormats_(IDWriteFontFace4* This,UINT16 glyph,UINT32 ppem_first,UINT32 ppem_last,DWRITE_GLYPH_IMAGE_FORMATS *formats) { +- return This->lpVtbl->GetGlyphImageFormats_(This,glyph,ppem_first,ppem_last,formats); ++static FORCEINLINE HRESULT IDWriteFontFace4_GetGlyphImageFormats(IDWriteFontFace4* This,UINT16 glyph,UINT32 ppem_first,UINT32 ppem_last,DWRITE_GLYPH_IMAGE_FORMATS *formats) { ++ return This->lpVtbl->GetGlyphImageFormats(This,glyph,ppem_first,ppem_last,formats); + } + static FORCEINLINE DWRITE_GLYPH_IMAGE_FORMATS IDWriteFontFace4_GetGlyphImageFormats(IDWriteFontFace4* This) { + return This->lpVtbl->GetGlyphImageFormats(This); +@@ -9020,7 +9020,7 @@ typedef struct IDWriteFontFace5Vtbl { + WINBOOL *are_local); + + /*** IDWriteFontFace4 methods ***/ +- HRESULT (STDMETHODCALLTYPE *GetGlyphImageFormats_)( ++ HRESULT (STDMETHODCALLTYPE *GetGlyphImageFormats)( + IDWriteFontFace5 *This, + UINT16 glyph, + UINT32 ppem_first, +@@ -9121,7 +9121,7 @@ interface IDWriteFontFace5 { + #define IDWriteFontFace5_AreCharactersLocal(This,characters,count,enqueue_if_not,are_local) (This)->lpVtbl->AreCharactersLocal(This,characters,count,enqueue_if_not,are_local) + #define IDWriteFontFace5_AreGlyphsLocal(This,glyphs,count,enqueue_if_not,are_local) (This)->lpVtbl->AreGlyphsLocal(This,glyphs,count,enqueue_if_not,are_local) + /*** IDWriteFontFace4 methods ***/ +-#define IDWriteFontFace5_GetGlyphImageFormats_(This,glyph,ppem_first,ppem_last,formats) (This)->lpVtbl->GetGlyphImageFormats_(This,glyph,ppem_first,ppem_last,formats) ++#define IDWriteFontFace5_GetGlyphImageFormats(This,glyph,ppem_first,ppem_last,formats) (This)->lpVtbl->GetGlyphImageFormats(This,glyph,ppem_first,ppem_last,formats) + #define IDWriteFontFace5_GetGlyphImageFormats(This) (This)->lpVtbl->GetGlyphImageFormats(This) + #define IDWriteFontFace5_GetGlyphImageData(This,glyph,ppem,format,data,context) (This)->lpVtbl->GetGlyphImageData(This,glyph,ppem,format,data,context) + #define IDWriteFontFace5_ReleaseGlyphImageData(This,context) (This)->lpVtbl->ReleaseGlyphImageData(This,context) +@@ -9270,8 +9270,8 @@ static FORCEINLINE HRESULT IDWriteFontFace5_AreGlyphsLocal(IDWriteFontFace5* Thi + return This->lpVtbl->AreGlyphsLocal(This,glyphs,count,enqueue_if_not,are_local); + } + /*** IDWriteFontFace4 methods ***/ +-static FORCEINLINE HRESULT IDWriteFontFace5_GetGlyphImageFormats_(IDWriteFontFace5* This,UINT16 glyph,UINT32 ppem_first,UINT32 ppem_last,DWRITE_GLYPH_IMAGE_FORMATS *formats) { +- return This->lpVtbl->GetGlyphImageFormats_(This,glyph,ppem_first,ppem_last,formats); ++static FORCEINLINE HRESULT IDWriteFontFace5_GetGlyphImageFormats(IDWriteFontFace5* This,UINT16 glyph,UINT32 ppem_first,UINT32 ppem_last,DWRITE_GLYPH_IMAGE_FORMATS *formats) { ++ return This->lpVtbl->GetGlyphImageFormats(This,glyph,ppem_first,ppem_last,formats); + } + static FORCEINLINE DWRITE_GLYPH_IMAGE_FORMATS IDWriteFontFace5_GetGlyphImageFormats(IDWriteFontFace5* This) { + return This->lpVtbl->GetGlyphImageFormats(This); diff --git a/projects/mingw-w64/config b/projects/mingw-w64/config index b0ace13..c5dde82 100644 --- a/projects/mingw-w64/config +++ b/projects/mingw-w64/config @@ -1,7 +1,7 @@ # vim: filetype=yaml sw=2 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' git_url: https://git.code.sf.net/p/mingw-w64/mingw-w64 -git_hash: 1b373beec6d07478ffba33726bb3bb21f32e4411 +git_hash: 660e09f3cb20f181b6d6435cb623d65a3922a063 version: '[% c("abbrev") %]' var: container:

1 0

[tor-browser-build/master] Merge remote-tracking branch 'boklmgl/bug_40252_v2'
by sysrqb＠torproject.org 22 Mar '21

22 Mar '21

commit a2c4754c5a73a6ed8ab00078189453668cd18d00 Merge: e32e80b bd6a4ce Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Mon Mar 22 20:53:13 2021 +0000 Merge remote-tracking branch 'boklmgl/bug_40252_v2' projects/binutils/config | 5 + projects/clang-source/build | 15 +-- projects/clang-source/config | 9 +- projects/clang-source/no-seh.patch | 145 -------------------------- projects/clang/build | 2 + projects/clang/config | 8 ++ projects/mingw-w64-clang/build | 4 + projects/mingw-w64-clang/config | 3 +- projects/mingw-w64-clang/mingw-dwrite_3.patch | 84 +++++++++++++++ projects/mingw-w64/config | 2 +- 10 files changed, 118 insertions(+), 159 deletions(-)

1 0

[tor-browser/tor-browser-78.9.0esr-10.0-1] fixup! Bug 10760: Integrate TorButton to TorBrowser core
by sysrqb＠torproject.org 22 Mar '21

22 Mar '21

commit 3538764d03de3bae5a06168b92b62f2145df0300 Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Mon Mar 22 20:46:24 2021 +0000 fixup! Bug 10760: Integrate TorButton to TorBrowser core --- toolkit/torproject/torbutton | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolkit/torproject/torbutton b/toolkit/torproject/torbutton index 74d22e9cc3cd..f03b9cb7c2f3 160000 --- a/toolkit/torproject/torbutton +++ b/toolkit/torproject/torbutton @@ -1 +1 @@ -Subproject commit 74d22e9cc3cd29bc5f676ffb17bfd3c05c59081a +Subproject commit f03b9cb7c2f36c9cbd1869e24a1662e7d2755b4e

1 0

[rbm/master] Bug 33991: Remove unused features
by gk＠torproject.org 22 Mar '21

22 Mar '21

commit 777bab010a84734a7428a836158745b1ca1c3eaa Author: Nicolas Vigier <boklm(a)torproject.org> Date: Sat Apr 25 14:32:27 2020 +0200 Bug 33991: Remove unused features For simplification, we remove some unused features. Those features can still be reimplemented externally if needed. For example, the "rbm rpm" command can be replaced with "rbm build --step rpm" and a definition of an rpm step in rbm.conf. We remove the following options: - version_command - rpm - rpmbuild - srpm - rpmspec - rpm_rel - debian_create - deb - deb_src - use_pbuilder - debian_revision And the following commands: - rpmspec - rpm - srpm - deb-src - deb - pkg - publish --- NEWS | 5 ++ doc/Makefile | 7 -- doc/clioptions_deb.asc | 7 -- doc/clioptions_publish.asc | 2 - doc/documentation.asc | 7 -- doc/features.asc | 13 ---- doc/options_deb.asc | 36 --------- doc/options_pkg.asc | 6 -- doc/options_publish.asc | 5 -- doc/options_rpm.asc | 25 ------ doc/rbm-deb-src.asc | 40 ---------- doc/rbm-deb.asc | 40 ---------- doc/rbm-pkg.asc | 33 -------- doc/rbm-publish.asc | 97 ------------------------ doc/rbm-rpm.asc | 36 --------- doc/rbm-rpmspec.asc | 25 ------ doc/rbm-srpm.asc | 36 --------- doc/rbm-usage.asc | 8 +- doc/rbm_cli.asc | 4 - doc/rbm_config.asc | 32 ++------ doc/rbm_input_files.asc | 14 ++-- doc/rbm_remote.asc | 9 +-- doc/rbm_steps.asc | 12 +-- doc/rbm_targets.asc | 35 ++++----- doc/rbm_templates.asc | 8 +- doc/rbm_tutorial.asc | 42 +++++++++-- doc/version.asc | 11 +-- lib/RBM.pm | 42 ++--------- lib/RBM/DefaultConfig.pm | 167 +---------------------------------------- packages/.gitignore | 2 - packages/keyring/boklm.gpg | Bin 3479 -> 0 bytes packages/projects/rbm/config | 10 --- packages/projects/rbm/rbm.spec | 29 ------- packages/rbm.conf | 2 - rbm | 59 +-------------- test.pl | 8 +- 36 files changed, 103 insertions(+), 811 deletions(-) diff --git a/NEWS b/NEWS index e54f988..9358901 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +- Remove the version_command, rpm, srpm, rpmbuild, rpmspec, rpm_rel, + debian_create, deb, deb_src, use_pbuilder, and debian_revision options, + as well as the rpmspec, rpm, srpm, deb-src, deb, pkg, and the publish + commands. All of these features can still be implemented externally. + - Add the target_append and target_prepend input_files options - Add the build_log_append option diff --git a/doc/Makefile b/doc/Makefile index 593ecba..8564a19 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -27,14 +27,7 @@ MAN1_ASC += rbm.asc MAN1_ASC += rbm-usage.asc MAN1_ASC += rbm-tar.asc MAN1_ASC += rbm-showconf.asc -MAN1_ASC += rbm-srpm.asc -MAN1_ASC += rbm-rpm.asc -MAN1_ASC += rbm-rpmspec.asc MAN1_ASC += rbm-fetch.asc -MAN1_ASC += rbm-deb-src.asc -MAN1_ASC += rbm-deb.asc -MAN1_ASC += rbm-pkg.asc -MAN1_ASC += rbm-publish.asc MAN1_ASC += rbm-build.asc MAN7_ASC += rbm_tutorial.asc diff --git a/doc/clioptions_deb.asc b/doc/clioptions_deb.asc deleted file mode 100644 index 69f8bdc..0000000 --- a/doc/clioptions_deb.asc +++ /dev/null @@ -1,7 +0,0 @@ - --use-pbuilder:: - --no-use-pbuilder:: - Whether to use pbuilder when building debian packages - - --debsign-keyid=<key id>:: - The id of the key to sign debian packages with. Set to 0 for no - signature. diff --git a/doc/clioptions_publish.asc b/doc/clioptions_publish.asc deleted file mode 100644 index 5f797cd..0000000 --- a/doc/clioptions_publish.asc +++ /dev/null @@ -1,2 +0,0 @@ ---publish-src-dir:: - The directory containing the files that are going to be published. diff --git a/doc/documentation.asc b/doc/documentation.asc index 82b5e62..4c5c9bc 100644 --- a/doc/documentation.asc +++ b/doc/documentation.asc @@ -15,12 +15,5 @@ rbm commands : - link:rbm-tar.html[rbm-tar(1)] - link:rbm-showconf.html[rbm-showconf(1)] - link:rbm-build.html[rbm-build(1)] -- link:rbm-srpm.html[rbm-srpm(1)] -- link:rbm-rpm.html[rbm-rpm(1)] -- link:rbm-rpmspec.html[rbm-rpmspec(1)] - link:rbm-fetch.html[rbm-fetch(1)] -- link:rbm-deb-src.html[rbm-deb-src(1)] -- link:rbm-deb.html[rbm-deb(1)] -- link:rbm-pkg.html[rbm-pkg(1)] -- link:rbm-publish.html[rbm-publish(1)] diff --git a/doc/features.asc b/doc/features.asc index f6f9116..ec68cbc 100644 --- a/doc/features.asc +++ b/doc/features.asc @@ -10,19 +10,6 @@ package creation:: Create binary or source packages from any git or mercurial revision using a simple command. -packages upload:: - Upload the packages that you have built to a repository and - update metadata. - -multi-os support:: - RPM and debian packages are currently supported. Support for - more packaging systems could easily be added. Using package - templates and the http://www.template-toolkit.org/[Template - Toolkit], you can make different packages for different - distributions using the same files. The syntax of this templating - system is much more clear and powerful than the RPM macros - syntax. - remote builds:: Build your software on a remote server with ssh, or in a local chroot using a different OS. For instance if you are using diff --git a/doc/options_deb.asc b/doc/options_deb.asc deleted file mode 100644 index 12b4a30..0000000 --- a/doc/options_deb.asc +++ /dev/null @@ -1,36 +0,0 @@ -debian_revision:: - The package revision used in debian packages. By default, when - the option +pkg_rel+ is defined, this is what is used. Otherwise - a revision containing the number of commits since the last git - tag, and the hash of the commit is used. - -deb_src:: - This is the script that is used to create the debian source - package. By default it will use the debian files listed in the - option +debian_files+ and create the source package with - dpkg-source. - -deb:: - This is the script that is used to create the debian packages. - By default it will use the debian files listed in the option - +debian_files+ and build the package using debuild or pdebuild - depending on whether the +use_pbuilder+ option is set. The - packages will be signed using the key defined in the option - +debsign_keyid+. - -debian_files:: - This is an array containing the files to create in the debian - directory. Each item in the array is an hash, with the following - two keys : +name+ is the file name in the debian directory of - the file to create, and +content+ is the content of the file. - The filename and content are processed as template, so for - instance if you want to store the content of a file in a separate - file, you can use the INCLUDE directive. - -use_pbuilder:: - If set to a true value, pbuilder will be used to build the - debian packages. - -debsign_keyid:: - This is the gpg key that will be used to sign the debian packages. - Set to 0 if you don't want to sign the packages. diff --git a/doc/options_pkg.asc b/doc/options_pkg.asc deleted file mode 100644 index 8542973..0000000 --- a/doc/options_pkg.asc +++ /dev/null @@ -1,6 +0,0 @@ -pkg_type:: - This is the name of the option that will be used by the +pkg+ - command as the script to build the package. This can be +rpm+ - or +deb+ for rpm and debian packages. This option is usually - set in distribution specific configuration as it depends on the - distribution being used. diff --git a/doc/options_publish.asc b/doc/options_publish.asc deleted file mode 100644 index 43d494c..0000000 --- a/doc/options_publish.asc +++ /dev/null @@ -1,5 +0,0 @@ -publish:: - This is the content of the script that is used to upload the - packages or files to a repository. This script will be executed - from the directory containing the files to publish. This option - has no default value. diff --git a/doc/options_rpm.asc b/doc/options_rpm.asc deleted file mode 100644 index b7a7489..0000000 --- a/doc/options_rpm.asc +++ /dev/null @@ -1,25 +0,0 @@ -rpm_rel:: - RPM package release number. The default is to use the option - +pkg_rel+ if defined, otherwise use a release number containing - the number of commits since the last git tag, and the hash of - the commit used. - -rpmspec:: - This is the content of the rpm spec file, used by the +rpm+ and - +srpm+ commands. The default is to include the template file named - +project.spec+ (with +project+ replaced by the project's name). - -rpmbuild:: - This is the content of the script to build a rpm or srpm. It is - using the +rpmbuild_action+ option to select the build action - ('-bs' to build a source package, or '-ba' to build all packages). - -rpm:: - This is the script that is used to build an rpm package, in the - +rpm+ command. By default it is using the +rpmbuild+ option with - the '-ba' action. - -srpm:: - This is the script that is used to build a source rpm package, - in the +srpm+ command. By default it is using the +rpmbuild+ - option with the '-bs' action. diff --git a/doc/rbm-deb-src.asc b/doc/rbm-deb-src.asc deleted file mode 100644 index d0212ea..0000000 --- a/doc/rbm-deb-src.asc +++ /dev/null @@ -1,40 +0,0 @@ -rbm-deb-src(1) -============== - -NAME ----- -rbm-deb-src - Create debian source package - - -SYNOPSIS --------- -[verse] -'rbm deb-src' <project> [options] - -DESCRIPTION ------------ - -Create a debian source package for the selected project. - -This is equivalent to running 'rbm build --step deb_src <project>'. - -CONFIGURATION OPTIONS ---------------------- - -The following configuration options are related to Debian packages : - -include::options_deb.asc[] - -See link:rbm_config.html[rbm_config(7)] for all other options. - -COMMAND LINE OPTIONS --------------------- -The command line options related to Debian packages are : - -include::clioptions_deb.asc[] - -See link:rbm_cli.html[rbm_cli(7)] for all other options. - -SEE ALSO --------- -link:rbm.html[rbm(1)] diff --git a/doc/rbm-deb.asc b/doc/rbm-deb.asc deleted file mode 100644 index 0544b89..0000000 --- a/doc/rbm-deb.asc +++ /dev/null @@ -1,40 +0,0 @@ -rbm-deb(1) -========== - -NAME ----- -rbm-deb - Build a debian package - - -SYNOPSIS --------- -[verse] -'rbm deb' <project> [options] - -DESCRIPTION ------------ - -Build a debian package. - -This is equivalent to running 'rbm build --step deb <project>'. - -CONFIGURATION OPTIONS ---------------------- - -The following configuration options are related to Debian packages : - -include::options_deb.asc[] - -See link:rbm_config.html[rbm_config(7)] for all other options. - -COMMAND LINE OPTIONS --------------------- -The command line options related to Debian packages are : - -include::clioptions_deb.asc[] - -See link:rbm_cli.html[rbm_cli(7)] for all other options. - -SEE ALSO --------- -link:rbm.html[rbm(1)] diff --git a/doc/rbm-pkg.asc b/doc/rbm-pkg.asc deleted file mode 100644 index 23ddaa9..0000000 --- a/doc/rbm-pkg.asc +++ /dev/null @@ -1,33 +0,0 @@ -rbm-pkg(1) -========== - -NAME ----- -rbm-pkg - Build a package - - -SYNOPSIS --------- -[verse] -'rbm pkg' <project> [options] - -DESCRIPTION ------------ - -Build a package for the selected project. Depending on configuration or -selected distribution, this can be an rpm, deb or other type of package. - -CONFIGURATION OPTIONS ---------------------- - -include::options_pkg.asc[] - -See link:rbm_config.html[rbm_config(7)] for all other options. - -COMMAND LINE OPTIONS --------------------- -See link:rbm_cli.html[rbm_cli(7)]. - -SEE ALSO --------- -link:rbm.html[rbm(1)] diff --git a/doc/rbm-publish.asc b/doc/rbm-publish.asc deleted file mode 100644 index f8ad50f..0000000 --- a/doc/rbm-publish.asc +++ /dev/null @@ -1,97 +0,0 @@ -rbm-publish(1) -============== - -NAME ----- -rbm-publish - Publish some packages or files - - -SYNOPSIS --------- -[verse] -'rbm publish' <project> [--publish-src-dir=<dir>] [options] - -DESCRIPTION ------------ - -Build a package and publish it. See link:rbm-pkg.html[rbm-pkg(1)] -for details about how the package is built. - -If the option '--publish-src-dir' is used, then the package is not built, -and instead the packages or files present in that directory are published. - -CONFIGURATION OPTIONS ---------------------- - -include::options_publish.asc[] - -See link:rbm_config.html[rbm_config(7)] for all other options. - -COMMAND LINE OPTIONS --------------------- -The command line options related to publish are : - -include::clioptions_publish.asc[] - -See link:rbm_cli.html[rbm_cli(7)] for all other options. - -EXAMPLES --------- - -In this example, we want to publish packages for Mageia and Debian. On -Mageia, the repository metadata are generated using 'genhdlist2'. On -Debian the new package is included in the repository using 'reprepro'. - -The rbm configuration looks like this : ----- -targets: - Mageia: - repo_dir: /path/to/mageia/repo - publish: | - #!/bin/sh - cp -va noarch/*.rpm [% c('repo_dir') %] - genhdlist2 [% c('repo_dir') %] - Debian: - repo_dir: /path/to/debian/repo - publish: | - #!/bin/sh - reprepro -b [% c('repo_dir') %] include \ - [% c('lsb_release/codename') %] *.changes ----- - -If we want to upload the packages on a remote server, before including -them on the repository, this can be done like this : - ----- -targets: - Debian: - repo_dir: /path/to/debian/repo - publish: | - #!/bin/sh - reprepro -b [% c('repo_dir') %] include \ - [% c('lsb_release/codename') %] *.changes -steps: - publish: - ssh_host: publish_server.somedomain - remote_exec: "[% c('remote_ssh') %]" ----- - -An alternative could be to generate the repository locally, and add an -rsync command at the end of the 'publish' script to upload the repository -to your publication server : - ----- -targets: - Debian: - local_repo: /path/to/debian/repo - publish: | - #!/bin/sh - reprepro -b [% c('local_repo') %] include \ - [% c('lsb_release/codename') %] *.changes - rsync -avH --delete [% c('local_repo') %] publish_server.somedomain:/repo ----- - -SEE ALSO --------- -link:rbm.html[rbm(1)], -link:rbm-pkg.html[rbm-pkg(1)] diff --git a/doc/rbm-rpm.asc b/doc/rbm-rpm.asc deleted file mode 100644 index 75f1886..0000000 --- a/doc/rbm-rpm.asc +++ /dev/null @@ -1,36 +0,0 @@ -rbm-rpm(1) -========== - -NAME ----- -rbm-rpm - Build an rpm package - - -SYNOPSIS --------- -[verse] -'rbm rpm' <project> [options] - -DESCRIPTION ------------ - -Build an rpm package for the selected project. - -This is equivalent to running 'rbm build --step rpm <project>'. - -CONFIGURATION OPTIONS ---------------------- - -The following configuration options are related to rpm packages : - -include::options_rpm.asc[] - -See link:rbm_config.html[rbm_config(7)] for all other options. - -COMMAND LINE OPTIONS --------------------- -See link:rbm_cli.html[rbm_cli(7)]. - -SEE ALSO --------- -link:rbm.html[rbm(1)] diff --git a/doc/rbm-rpmspec.asc b/doc/rbm-rpmspec.asc deleted file mode 100644 index ea06427..0000000 --- a/doc/rbm-rpmspec.asc +++ /dev/null @@ -1,25 +0,0 @@ -rbm-rpmspec(1) -============== - -NAME ----- -rbm-rpmspec - Create a rpm spec file - - -SYNOPSIS --------- -[verse] -'rbm rpmspec' <project> [options] - -DESCRIPTION ------------ - -Create an rpm spec file for the selected project. - -OPTIONS -------- -See link:rbm_cli.html[rbm_cli(7)]. - -SEE ALSO --------- -link:rbm.html[rbm(1)] diff --git a/doc/rbm-srpm.asc b/doc/rbm-srpm.asc deleted file mode 100644 index bc2a28c..0000000 --- a/doc/rbm-srpm.asc +++ /dev/null @@ -1,36 +0,0 @@ -rbm-srpm(1) -=========== - -NAME ----- -rbm-srpm - Create source rpm package - - -SYNOPSIS --------- -[verse] -'rbm srpm' <project> [options] - -DESCRIPTION ------------ - -Create a source rpm package for the select project. - -This is equivalent to running 'rbm build --step srpm <project>'. - -CONFIGURATION OPTIONS ---------------------- - -The following configuration options are related to rpm packages : - -include::options_rpm.asc[] - -See link:rbm_config.html[rbm_config(7)] for all other options. - -COMMAND LINE OPTIONS --------------------- -See link:rbm_cli.html[rbm_cli(7)]. - -SEE ALSO --------- -link:rbm.html[rbm(1)] diff --git a/doc/rbm-usage.asc b/doc/rbm-usage.asc index bd74d2f..400acd0 100644 --- a/doc/rbm-usage.asc +++ b/doc/rbm-usage.asc @@ -22,10 +22,10 @@ EXAMPLES -------- ---- -$ rbm usage rpm - => man rbm-rpm -$ rbm deb --help - => man rbm-deb +$ rbm usage build + => man rbm-build +$ rbm fetch --help + => man rbm-fetch $ rbm --help tar => man rbm-tar ---- diff --git a/doc/rbm_cli.asc b/doc/rbm_cli.asc index 0e6828c..03ed511 100644 --- a/doc/rbm_cli.asc +++ b/doc/rbm_cli.asc @@ -71,10 +71,6 @@ The following options can be set on the command line : include::clioptions_tar.asc[] -include::clioptions_deb.asc[] - -include::clioptions_publish.asc[] - SEE ALSO -------- link:rbm.html[rbm(1)], diff --git a/doc/rbm_config.asc b/doc/rbm_config.asc index 111b115..4c65a73 100644 --- a/doc/rbm_config.asc +++ b/doc/rbm_config.asc @@ -181,14 +181,6 @@ version:: Version number of the software. This is used to create the tarball, and as the package version number. -version_command:: - A command to run in the checked out source tree to determine - the version, if the +version+ option is not set. The command - should print the version on stdout. - -pkg_rel:: - Package release number. - distribution:: The name of the distribution for which you wish to build a package. The syntax is +distribution-release+. This value is used by the @@ -242,9 +234,9 @@ input_files_paths:: timestamp:: This is the UNIX timestamp, set as modification time on files - created such as the sources tarball and rpm spec file. The - default is to use the commit time of the commit used. If set to - 0 it will use the current time. + created such as the sources tarball. The default is to use the + commit time of the commit used. If set to 0 it will use the + current time. notmpl:: An array containing a list of options that should not be @@ -253,29 +245,21 @@ notmpl:: step:: The value of this option is the name of the build script we are - going to be running ('deb' if building a Debian package, 'rpm' - if building an rpm, etc ...). This can be useful in the - +input_files+ definition, if you want to enable an input file - only for some type of package. This option should be used read - only. + going to be running (by default 'build', but you could have an + other one for publishing your build, or for different types of + packaging). This option should be used read only. If you want + to change it, use the --step command line option, or the + +pkg_type+ option. steps:: The steps definitions. See link:rbm_steps.html[rbm_steps(7)] for details. -include::options_rpm.asc[] - -include::options_deb.asc[] - -include::options_pkg.asc[] - build:: This is the content of the build script used by the +build+ command. The default is to include the template file named +build+. -include::options_publish.asc[] - include::options_misc.asc[] In addition to the configuration options listed here, you are free to diff --git a/doc/rbm_input_files.asc b/doc/rbm_input_files.asc index de47638..55aaebf 100644 --- a/doc/rbm_input_files.asc +++ b/doc/rbm_input_files.asc @@ -68,12 +68,14 @@ project:: The name of an other project to build, to create the file. The content of the file descriptor is used as options for the build of this project. You can add config options such as +version+, - +git_hash+ or +target+ to configure the build. The build is run - with an equivalent of 'rbm pkg', so you should define the - +pkg_type+ option if you want to select the type of build. The - command line options are not forwarded. The name of the current - project is available to that other project in the +origin_project+ - option. + +git_hash+ or +target+ to configure the build. The command line + options are not forwarded. The name of the current project is + available to that other project in the +origin_project+ option. + +pkg_type:: + When the input file is an other project, this option can be + used to select a different type of build (which we call a step). + This is equivalent to the --step command line option. target:: An array defining the build targets for the selected project. If this diff --git a/doc/rbm_remote.asc b/doc/rbm_remote.asc index 479062a..81de98f 100644 --- a/doc/rbm_remote.asc +++ b/doc/rbm_remote.asc @@ -95,15 +95,12 @@ REMOTE BUILD WITH SSH --------------------- Some predefined +remote_*+ templates are available and can be enabled -by setting the +remote_ssh+ option to 1. For instance if you want to -build your debian packages on a remote node with ssh, add the following -lines to your configuration : +by setting the +remote_ssh+ option to 1. For instance you can add the +following lines to your configuration: ---- ssh_host: some_hostname -steps: - deb_src: - remote_ssh: 1 +remote_ssh: 1 ---- The following options will be used: diff --git a/doc/rbm_steps.asc b/doc/rbm_steps.asc index b02891b..191dd19 100644 --- a/doc/rbm_steps.asc +++ b/doc/rbm_steps.asc @@ -25,9 +25,10 @@ CONFIGURATION The first way to have a different option value depending on the curernt step is to use the IF/ELSE template directives and the +step+ option. -The +step+ option contains the name of the current step. For instance -if you want to use a different git tag or branch for rpm and debian -packages, you could do: +The +step+ option contains the name of the current step. For instance, +assuming that you defined rpm and deb steps for building rpm and +debian packages, if you want to use a different git tag or branch +depending on the type of package, you could do: ---- git_hash: | [% IF c('step') == 'rpm' -%] @@ -85,11 +86,10 @@ steps: If the value of a step is not a hash containing options, but a string, then this name is used as step name. This can be used to alias a step configuration to an other step. For instance you usually want to use -the same options for the rpm and srpm scripts. The default configuration -includes the following steps aliases: +the same options for the rpm and srpm scripts, which can be done with +the following step alias: - srpm: rpm -- deb-src: deb PRE AND POST SCRIPTS diff --git a/doc/rbm_targets.asc b/doc/rbm_targets.asc index fdcccca..d50aff3 100644 --- a/doc/rbm_targets.asc +++ b/doc/rbm_targets.asc @@ -31,25 +31,25 @@ target. As an example we could define the following targets : - the 'dev' target, for daily development, from the master branch. You - don't to use pbuilder, so you can have your package built more quickly. - You don't want to sign the packages. And you want this target to be the - default. + want to automatically fetch from the git repository before each build, + and you want to check that the commit used is signed by a key from the + developers.gpg keyring. - the 'release' target, for building packages for a new release. You - want to use pbuilder to make sure it is built in a clean chroot. You - only want to build them from gpg signed tags. You want to sign the - packages with a specific key. + only want to build them from gpg signed tags, signed by the key from + the release.gpg keyring. In such case, the following targets configuration could be used : ---- targets: dev: git_hash: master - use_pbuilder: 0 + fetch: 1 + commit_gpg_id: 1 + gpg_keyring: developers.gpg release: - use_pbuilder: 1 - debsign_keyid: 123456 tag_gpg_id: 1 + gpg_keyring: release.gpg ---- If the value of a target is not a hash containing options, but a string @@ -65,11 +65,12 @@ targets: notarget: dev dev: git_hash: master - use_pbuilder: 0 + fetch: 1 + commit_gpg_id: 1 + gpg_keyring: developers.gpg release: - use_pbuilder: 1 - debsign_keyid: 123456 tag_gpg_id: 1 + gpg_keyring: release.gpg ---- In this example we extend the 'release' target as 'release_1' to add a @@ -80,9 +81,8 @@ targets: - git_hash_version_1 - release release: - use_pbuilder: 1 - debsign_keyid: 123456 tag_gpg_id: 1 + gpg_keyring: release.gpg git_hash_version_1: git_hash: version-1.0 ---- @@ -96,15 +96,16 @@ times, and in this case they will be used in the order given on the command line. If no target is selected, then the 'notarget' target is used. -For example, if you want to build a debian package for 'dev' target : +For example, if you want to build a debian package (assuming that you +defined a 'deb' step for that) for the 'dev' target: ---- -$ rbm deb --target dev +$ rbm build --step deb --target dev ---- If you want to build a debian package for the 'release' target, using the '0.1.2.3' tag: ---- -$ rbm deb --target release --git-hash 0.1.2.3 +$ rbm build --step deb --target release --git-hash 0.1.2.3 ---- Sometime, an option in one project will use the value of an option from diff --git a/doc/rbm_templates.asc b/doc/rbm_templates.asc index 2f72c94..4c1a619 100644 --- a/doc/rbm_templates.asc +++ b/doc/rbm_templates.asc @@ -32,13 +32,7 @@ template files can be added in the directory +projects_dir/common+, to be included from any of the other templates. There are different template files : -By default, the following template files are used : - -- the rpm spec file template, named +project.spec+ (replacing +project+ - with the project's name). This is used when you use the +rpmspec+, - +srpm+ +rpm+, or +build+ commands. This creates the +rpmspec+ option. - If you don't want to use this file, just replace the +rpmspec+ option - by something else. +By default, the following template file is used, but you can add more: - the build script template, named +build+. This template is used to create a build script, that is executed when you use the +build+ diff --git a/doc/rbm_tutorial.asc b/doc/rbm_tutorial.asc index 9471034..2728478 100644 --- a/doc/rbm_tutorial.asc +++ b/doc/rbm_tutorial.asc @@ -82,16 +82,18 @@ the software, for any git commit. By default, rbm will use the latest tag on which a commit is based as the version number. Sometimes it works, when the project uses version numbers as tags, but this is not always the case. Alternatively, you can -define the 'version_command' option with a shell script or command that +define a 'var/version_command' option with a shell script or command that will print the version number. -For tor, we will use the 'version_command' option. The version of the +For tor, we will use a 'var/version_command' option. The version of the software is defined in the ChangeLog file, so we will use a simple command to get it. The tor config file now look like this : ---- git_url: https://git.torproject.org/tor.git -version_command: perl -ne 'if (m/^Changes in version ([^-]+)-.*$/) { print "$1\n"; exit }' < ChangeLog +version: '[% exec(c("var/version_command")) %]' +var: + version_command: perl -ne 'if (m/^Changes in version ([^-]+)-.*$/) { print "$1\n"; exit }' < ChangeLog ---- Using the 'showconf' command, we can check the value of the 'version' @@ -123,17 +125,41 @@ Created /home/boklm/rbm/out/tor-0.2.4.17.tar.xz Building an rpm package ----------------------- -TODO +In order to build rpm packages, you will need to define an rpm step +(see link:rbm_steps.html[rbm_steps(7)]) in rbm.conf. + +The rbm.conf file should define an +rpm+ option containing the commands +to build an rpm package: +---- +rpm: | + #!/bin/bash + [ here the commands for building an rpm package ] +---- + +Unfortunately, this part of the tutorial, with details about the +definition of the rpm step has not been written yet. + +You should be able to start a build with the 'build' command and +the '--step' option: +---- +$ rbm build --step rpm tor +---- Building a debian package ------------------------- -TODO +Similarly to the rpm package, building debian packages will require that +you define a deb step in rbm.conf. This part of the tutorial has not +been written yet. -Using a remote build server ---------------------------- +Using a build container +----------------------- -TODO +Using the remote options (see link:rbm_remote.html[rbm_remote(7)]) you +can do the builds inside containers. Unfortunately this part of the +tutorial has not been written yet. However you can look at the +Tor Browser build repository for an example: +https://gitweb.torproject.org/builders/tor-browser-build.git/ SEE ALSO -------- diff --git a/doc/version.asc b/doc/version.asc index 9082ff0..bdbc71a 100644 --- a/doc/version.asc +++ b/doc/version.asc @@ -1,13 +1,4 @@ The version of the software used to create the tarball and the packages -can be set with the 'version' option. If the version is not explicitely -set, then it is determined automatically in the following way : - -- If the +version_command+ option is set, then the value of this option - is run in the checked out source tree, and the output is used as the - version. - -- If the +version_command+ is not set, or if running the command failed, - then the most recent tag (as returned by git-describe) is used as - version. +should be set with the 'version' option. diff --git a/lib/RBM.pm b/lib/RBM.pm index 2666e72..1294b4e 100644 --- a/lib/RBM.pm +++ b/lib/RBM.pm @@ -686,17 +686,6 @@ sub process_template { return $output; } -sub rpmspec { - my ($project, $dest_dir) = @_; - $dest_dir //= create_dir(rbm_path(project_config($project, 'output_dir'))); - valid_project($project); - my $timestamp = project_config($project, 'timestamp'); - my $rpmspec = project_config($project, 'rpmspec') - || exit_error "Undefined config for rpmspec"; - path("$dest_dir/$project.spec")->spew_utf8($rpmspec); - utime $timestamp, $timestamp, "$dest_dir/$project.spec" if $timestamp; -} - sub projectslist { keys %{$config->{projects}}; } @@ -1020,20 +1009,13 @@ sub build_run { valid_project($project); $options = { %$options, build_id => Data::UUID->new->create_str }; my $old_cwd = getcwd; - my $srcdir = project_config($project, 'build_srcdir', $options); - my $use_srcdir = $srcdir; - my $tmpdir = File::Temp->newdir(get_tmp_dir($project, $options) + my $srcdir = File::Temp->newdir(get_tmp_dir($project, $options) . '/rbm-XXXXX'); my @cfiles; - if ($use_srcdir) { - @cfiles = ($srcdir); - } else { - $srcdir = $tmpdir->dirname; - my $tarfile = maketar($project, $options, $srcdir); - push @cfiles, $tarfile if $tarfile; - push @cfiles, copy_files($project, $srcdir); - push @cfiles, input_files('copy', $project, $options, $srcdir); - } + my $tarfile = maketar($project, $options, $srcdir); + push @cfiles, $tarfile if $tarfile; + push @cfiles, copy_files($project, $srcdir); + push @cfiles, input_files('copy', $project, $options, $srcdir); my ($remote_tmp_src, $remote_tmp_dst, %build_script); my @scripts = ('pre', $script_name, 'post'); my %scripts_root = ( pre => 1, post => 1); @@ -1083,7 +1065,7 @@ sub build_run { goto EXIT; } @scripts = grep { $build_script{$_} } @scripts; - push @cfiles, @scripts unless $use_srcdir; + push @cfiles, @scripts; foreach my $s (@scripts) { path("$srcdir/$s")->spew_utf8($build_script{$s}); chmod 0700, "$srcdir/$s"; @@ -1185,17 +1167,5 @@ sub build_pkg { build_run($project, project_config($project, 'pkg_type', $options), $options); } -sub publish { - my ($project) = @_; - project_config($project, 'publish', { error_if_undef => 1 }); - my $publish_src_dir = project_config($project, 'publish_src_dir'); - if (!$publish_src_dir) { - $publish_src_dir = File::Temp->newdir(get_tmp_dir($project) - . '/rbm-XXXXXX'); - build_pkg($project, { norec => { output_dir => $publish_src_dir } }); - } - build_run($project, 'publish', { build_srcdir => $publish_src_dir }); -} - 1; # vim: expandtab sw=4 diff --git a/lib/RBM/DefaultConfig.pm b/lib/RBM/DefaultConfig.pm index a4c1e5c..c19d160 100644 --- a/lib/RBM/DefaultConfig.pm +++ b/lib/RBM/DefaultConfig.pm @@ -15,23 +15,6 @@ use Cwd qw(getcwd); use IO::CaptureOutput qw(capture_exec); use File::Temp; -sub git_describe { - my ($project, $options) = @_; - my $git_hash = RBM::project_config($project, 'git_hash', $options) - || RBM::exit_error('No git_hash specified'); - my %res; - $RBM::config->{projects}{$project}{describe} = {}; - my $old_cwd = getcwd; - RBM::git_clone_fetch_chdir($project, $options); - my ($stdout, $stderr, $success, $exit_code) - = capture_exec('git', 'describe', '--long', $git_hash); - if ($success) { - @res{qw(tag tag_reach hash)} = $stdout =~ m/^(.+)-(\d+)-g([^-\n]+)$/; - } - chdir($old_cwd); - return $success ? \%res : undef; -} - sub lsb_release { my ($project, $options) = @_; my $distribution = RBM::project_config($project, 'distribution', $options); @@ -123,12 +106,11 @@ our %default_config = ( git_clone_dir => 'git_clones', hg_clone_dir => 'hg_clones', fetch => 'if_needed', - rpmspec => '[% SET tmpl = project _ ".spec"; INCLUDE $tmpl -%]', + pkg_type => 'build', build => '[% INCLUDE build -%]', build_log => '-', build_log_append => '1', notmpl => [ qw(projects_dir) ], - describe => \&git_describe, abbrev_length => '12', abbrev => '[% IF c("git_url"); @@ -155,56 +137,7 @@ our %default_config = ( return '946684800'; }, debug => 0, - version => <<END, -[%- - IF c('version_command'); - exec(c('version_command')); - ELSIF c(['describe', 'tag']); - c(['describe', 'tag']); - ELSE; - exit_error('No version specified'); - END; --%] -END -#### -#### -#### - pkg_type => 'build', - rpm => '[% c("rpmbuild", { rpmbuild_action => "-ba" }) %]', - srpm => '[% c("rpmbuild", { rpmbuild_action => "-bs" }) %]', -#### -#### -#### - rpmbuild => <<END, -[% USE date -%] -#!/bin/sh -set -e -x -srcdir=\$(pwd) -cat > '[% project %].spec' << 'RBM_END_RPM_SPEC' -[% c('rpmspec') %] -RBM_END_RPM_SPEC -touch -m -t [% date.format(c('timestamp'), format = '%Y%m%d%H%M') %] [% project %].spec -rpmbuild [% c('rpmbuild_action', {error_if_undef => 1}) %] --define "_topdir \$srcdir" \\ - --define "_sourcedir \$srcdir" \\ - --define '_srcrpmdir [% dest_dir %]' \\ - --define '_rpmdir [% dest_dir %]' \\ - --define '_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm' \\ - "\$srcdir/[% project %].spec" -END -#### -#### -#### - rpm_rel => <<OPT_END, -[%- - IF c('pkg_rel').defined; - GET c('pkg_rel'); - ELSIF c('describe/tag_reach'); - GET '1.' _ c('describe/tag_reach') _ '.g' _ c('describe/hash'); - ELSE; - GET '1.g' _ c('abbrev'); - END; --%] -OPT_END + version => "[%- exit_error('No version specified'); -%]", #### #### #### @@ -224,87 +157,6 @@ exec [% c('gpg_bin') %] [% c('gpg_args') %] --with-fingerprint [% gpg_kr %] "\$@ GPGEND #### #### -#### - debian_create => <<DEBEND, -[%- - FOREACH f IN c('debian_files'); - GET 'cat > ' _ tmpl(f.name) _ " << 'END_DEBIAN_FILE'\n"; - GET tmpl(f.content); - GET "\nEND_DEBIAN_FILE\n\n"; - END; --%] -DEBEND -#### -#### -#### - deb_src => <<DEBEND, -#!/bin/sh -set -e -x -[% SET tarfile = project _ '-' _ c('version') _ '.tar.' _ c('compress_tar') -%] -tar xvf [% tarfile %] -mv [% tarfile %] [% dest_dir %]/[% project %]_[% c('version') %].orig.tar.[% c('compress_tar') %] -cd [% project %]-[% c('version') %] -builddir=\$(pwd) -mkdir debian debian/source -cd debian -[% c('debian_create') %] -cd [% dest_dir %] -dpkg-source -b "\$builddir" -DEBEND -#### -#### -#### - deb => <<DEBEND, -#!/bin/sh -set -e -x -[% SET tarfile = project _ '-' _ c('version') _ '.tar.' _ c('compress_tar') -%] -tar xvf [% tarfile %] -mv [% tarfile %] [% project %]_[% c('version') %].orig.tar.[% c('compress_tar') %] -cd [% project %]-[% c('version') %] -builddir=\$(pwd) -mkdir debian debian/source -cd debian -[% c('debian_create') %] -cd .. -ls .. -[% IF c('debsign_keyid'); - pdebuild_sign = '--debsign-k ' _ c('debsign_keyid'); - debuild_sign = '-k' _ c('debsign_keyid'); -ELSE; - pdebuild_sign = ''; - debuild_sign = '-uc -us'; -END -%] -[% IF c('use_pbuilder') -%] -pdebuild [% pdebuild_sign %] --buildresult [% dest_dir %] -[% ELSE -%] -debuild [% debuild_sign %] -cd .. -rm -f build -for file in * -do - if [ -f "\$file" ] - then - mv "\$file" [% dest_dir %] - fi -done -[% END -%] -DEBEND -#### -#### -#### - debian_revision => <<OPT_END, -[%- -IF c('pkg_rel'); - GET c('pkg_rel').defined; -ELSIF c('describe/tag_reach'); - GET '1.' _ c('describe/tag_reach') _ '~g' _ c('describe/hash'); -ELSE; - GET '1'; -END; --%] -OPT_END -#### -#### #### ssh_remote_exec => <<OPT_END, [%- @@ -519,19 +371,6 @@ OPT_END #### #### lsb_release => \&lsb_release, - pkg_type => sub { - my ($project, $options) = @_; - my $distro = RBM::project_config($project, 'lsb_release/id', $options); - my %pkg_types = qw( - Fedora rpm - CentOS rpm - Mageia rpm - openSuSe rpm - Debian deb - Ubuntu deb - ); - return $pkg_types{$distro}; - }, install_package => sub { my ($project, $options) = @_; my $distro = RBM::project_config($project, 'lsb_release/id', $options); @@ -606,8 +445,6 @@ ZIP_END input_files_id => sub { RBM::input_files('input_files_id', @_); }, input_files_paths => sub { RBM::input_files('getfpaths', @_); }, steps => { - srpm => 'rpm', - 'deb-src' => 'deb', }, suexec => 'sudo -- [% c("suexec_cmd") %]', hg => 'hg [% c("hg_opt") %]', diff --git a/packages/.gitignore b/packages/.gitignore deleted file mode 100644 index 16833ae..0000000 --- a/packages/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -git_clones -out diff --git a/packages/keyring/boklm.gpg b/packages/keyring/boklm.gpg deleted file mode 100644 index c878f59..0000000 Binary files a/packages/keyring/boklm.gpg and /dev/null differ diff --git a/packages/projects/rbm/config b/packages/projects/rbm/config deleted file mode 100644 index 44e497c..0000000 --- a/packages/projects/rbm/config +++ /dev/null @@ -1,10 +0,0 @@ -git_url: 'file://[% c("basedir") %]/..' -git_hash: master -compress_tar: xz -commit_gpg_id: 1 -summary: Tool to generate packages from a git repository, using templates -description: | - rbm is a tool to generate some packages from a git repository, using - package templates. -url: http://rbm.boklm.eu/ -gpg_keyring: boklm.gpg diff --git a/packages/projects/rbm/rbm.spec b/packages/projects/rbm/rbm.spec deleted file mode 100644 index 37b2941..0000000 --- a/packages/projects/rbm/rbm.spec +++ /dev/null @@ -1,29 +0,0 @@ -Name: [% project %] -Version: [% c('version') %] -Release: [% c('rpm_rel') %]%{?dist} -Source: %{name}-%{version}.tar.[% c('compress_tar') %] -Summary: [% c('summary') %] -URL: [% c('url') %] -License: CC0 -Group: Text tools -BuildRequires: asciidoc -BuildArch: noarch -%description -[% c('description') -%] - -%prep -%setup -q - -%build - -%install -make perldir=%{perl_vendorlib} DESTDIR=%{buildroot} install - -%files -%doc README.md COPYING -%{_bindir}/%{name} -%{perl_vendorlib}/RBM.pm -%{perl_vendorlib}/RBM/DefaultConfig.pm -%{_mandir}/man1/rbm.1* -%{_mandir}/man1/rbm-*.1* -%{_mandir}/man7/rbm_*.7* diff --git a/packages/rbm.conf b/packages/rbm.conf deleted file mode 100644 index b34b34d..0000000 --- a/packages/rbm.conf +++ /dev/null @@ -1,2 +0,0 @@ -compress_tar: xz -output_dir: "out/[% project %]" diff --git a/rbm b/rbm index 6b31185..764141e 100755 --- a/rbm +++ b/rbm @@ -21,38 +21,10 @@ my %actions = ( run => \&tar, descr => 'Create source tarball', }, - rpmspec => { - run => \&rpmspec, - descr => 'Create rpm spec file', - }, - rpm => { - run => sub { build_script('rpm', @_) }, - descr => 'Build an rpm package', - }, - srpm => { - run => sub { build_script('srpm', @_) }, - descr => 'Create source rpm file', - }, - 'deb-src' => { - run => sub { build_script('deb_src', @_) }, - descr => 'Create debian source package', - }, - 'deb' => { - run => sub { build_script('deb', @_) }, - descr => 'Create debian package', - }, build => { run => sub { build_script('build', @_) }, descr => 'Build project with a custom build script', }, - pkg => { - run => \&pkg, - descr => 'Build a package', - }, - publish => { - run => \&publish, - descr => 'Publish a package', - }, showconf => { run => \&show_conf, descr => 'Show configuration', @@ -91,10 +63,9 @@ sub usageexit { sub set_options { my @options = qw(distribution=s version=s tag-gpg-id=s@ commit-gpg-id=s@ projects-dir=s git-clone-dir=s git-hash=s output-dir=s - compress_tar=s pkg-rel=s timestamp=i fetch! - gpg-keyring=s gpg-keyring-dir=s gpg-args=s gpg-bin=s - sysconf-file=s debsign-keyid=s use-pbuilder! - step=s target=s@ publish-src-dir=s debug! hg-clone-dir=s + compress_tar=s timestamp=i fetch! gpg-keyring=s + gpg-keyring-dir=s gpg-args=s gpg-bin=s sysconf-file=s + step=s target=s@ debug! hg-clone-dir=s hg-hash=s localconf-file=s build-log=s); my %val; Getopt::Long::GetOptionsFromArray(\@_, \%val, @options) || exit 1; @@ -158,14 +129,6 @@ sub tar { RBM::maketar($args->[0]); } -sub rpmspec { - usageexit($_[0]) unless @_ >= 2; - shift; - my $args = set_options(@_); - usageexit('rpmspec') unless @$args == 1; - RBM::rpmspec($args->[0]); -} - sub print_projects { usageexit($_[0]) unless @_ == 1; print join("\n", RBM::projectslist()), "\n"; @@ -181,22 +144,6 @@ sub build_script { RBM::build_pkg($args->[0], { pkg_type => $script_name }); } -sub pkg { - usageexit($_[0]) unless @_ >= 2; - my $cmd = shift; - my $args = set_options(@_); - usageexit($cmd) unless @$args == 1; - RBM::build_pkg($args->[0]); -} - -sub publish { - usageexit($_[0]) unless @_ >= 2; - my $cmd = shift; - my $args = set_options(@_); - usageexit($cmd) unless @$args == 1; - RBM::publish($args->[0]); -} - if (@ARGV == 0 || !$actions{$ARGV[0]}) { usage(); exit 1; diff --git a/test.pl b/test.pl index dfca8ee..6c99cf0 100755 --- a/test.pl +++ b/test.pl @@ -134,14 +134,14 @@ my @tests = ( expected => 'aZa AZa aZa', }, { - name => 'srpm step', - step => 'srpm', + name => 'rpm step', + step => 'rpm', config => [ 'c', 'option_rpm' ], expected => '1', }, { - name => 'deb-src step', - step => 'deb-src', + name => 'deb step', + step => 'deb', config => [ 'c', 'option_deb' ], expected => '1', },

1 0

[tor-browser-spec/master] Update NETWORK_AUDIT_RUBRIC
by gk＠torproject.org 19 Mar '21

19 Mar '21

commit 6735a8b4b3fda67088089c5a06b8ba25d2f89a44 Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Tue Feb 9 19:29:32 2021 +0000 Update NETWORK_AUDIT_RUBRIC --- audits/NETWORK_AUDIT_RUBRIC | 93 +++++++++++---------------------------------- 1 file changed, 22 insertions(+), 71 deletions(-) diff --git a/audits/NETWORK_AUDIT_RUBRIC b/audits/NETWORK_AUDIT_RUBRIC index b58bb49..a755b64 100644 --- a/audits/NETWORK_AUDIT_RUBRIC +++ b/audits/NETWORK_AUDIT_RUBRIC @@ -1,88 +1,39 @@ -`git diff esrA esrB` and then go over all the changes containing the -below mentioned potentially dangerous calls and features. Grep the diff for -the following strings and examine surrounding usage. +============ General ============= -=============== Native DNS Portion ============= +The audit begins at the commit hash where the previous audit ended. Use +code_audit.sh for creating the diff and highlighting potentially problematic +code. The audit is scoped to a specific language (currently C/C++, Rust, +Java/Kotlin, and Javascript). -PR_GetHostByName -PR_GetIPNodeByName -PR_GetAddrInfoByName -PR_StringToNetAddr (itself is good as it passes AI_NUMERICHOST to getaddrinfo. No resolution.) +The output includes the entire patch where the new problematic code was +introduced. Search for "XXX MATCH XXX" to find the next potential violation. -MDNS -TRR (DNS Trusted Recursive Resolver) +code_audit.sh contains the list of known problematic APIs. New usage of these +functions are documented and analyzed in this audit. -Direct Paths to DNS resolution: -nsDNSService::Resolve -nsDNSService::AsyncResolve -nsHostResolver::ResolveHost +============ Firefox General Portion ============= -============ Misc Socket Portion ============== +Start: ${previous_base_commit_hash} # ${prev_tag} +End: ${next_base_commit_hash} # ${next_tag} -SOCK_ -SOCKET_ -_SOCKET -UDPSocket -TCPSocket - PR_NewTCPSocket - AsyncTCPSocket +============ Application Services Portion ============= -Misc PR_Socket +Start: ${previous_base_commit_hash} # ${prev_tag} +End: ${next_base_commit_hash} # ${next_tag} -=========== Misc XPCOM Portion ================ +============ Android Components Portion ============= -Misc XPCOM (including commands for pre-diff review approach) - *SocketProvider - grep -R udp-socket . - grep -R tcp-socket . - grep for tcpsocket - grep -R "NS_" | grep SOCKET | grep "_C" - grep -R "@mozilla.org/network/" . | grep socket | grep -v udp-socket +Start: ${previous_base_commit_hash} # ${prev_tag} +End: ${next_base_commit_hash} # ${next_tag} -============ Rust Portion ================ +============ Fenix Portion ============= -Rust - - XXX: What do we grep for here? Or do we rely on Ritter's compile-time tool? - - Check for new sendmsg and recvmsg usage - -============ Android Portion ============= - -Android Java calls - - URLConnection - - XXX: getInputStream? other methods? - - HttpURLConnection - - UrlConnectionDownloader - - ch.boye.httpclientandroidlib.impl.client.* (look for execute() calls) - - grep -n openConnection\( mobile/android/thirdparty/ - - java.net.URL -- has SEVERAL proxy bypass URL fetching methods :/ - - java.net - - javax.net - - ch.boye.httpclientandroidlib.conn.* (esp ssl) - - ch.boye.httpclientandroidlib.impl.conn.* (esp ssl) - - Sudden appearance of thirdparty libs: - - OkHttp - - Retrofit - - Glide - - com.amitshekhar.android - - IntentHelper - - openUriExternal (can come from GeckoAppShell too) - - getHandlersForMimeType - - getHandlersForURL - - getHandlersForIntent - - android.content.Intent - too common; instead find launch methods: - - startActivity - - startActivities - - sendBroadcast - - sendOrderedBroadcast - - startService - - bindService - - android.app.PendingIntent - - android.app.DownloadManager - - ActivityHandlerHelper.startIntentAndCatch +Start: ${previous_base_commit_hash} # ${prev_tag} +End: ${next_base_commit_hash} # ${next_tag} ============ Regression/Prior Vuln Review ========= Review proxy bypass bugs; check for new vectors to look for: - - https://trac.torproject.org/projects/tor/query?keywords=~tbb-proxy + - https://gitlab.torproject.org/groups/tpo/applications/-/issues?scope=all&ut… - Look for new features like these. Especially external app launch vectors

1 0

[tor-browser-spec/master] Bug 40010: Generalize audit script
by gk＠torproject.org 19 Mar '21

19 Mar '21

commit 68566ac4e7021456026bf057e47a00fa8513d2cf Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Thu Jan 21 21:29:22 2021 +0000 Bug 40010: Generalize audit script --- audits/java_audit.sh | 206 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 142 insertions(+), 64 deletions(-) diff --git a/audits/java_audit.sh b/audits/java_audit.sh old mode 100644 new mode 100755 index b1183eb..3586470 --- a/audits/java_audit.sh +++ b/audits/java_audit.sh @@ -1,90 +1,168 @@ #!/bin/bash -e -if [ $# -ne 3 ]; then - echo "usage: <path/to/repo> <old commit> <new commit>" +if [ $# -ne 4 ]; then + echo "usage: <path/to/repo> <lang> <old commit> <new commit>" exit 1 fi REPO_DIR=$1 -OLD=$2 -NEW=$3 - -SCOPE="java" # string: this is the java audit -EXT="java kt" +SCOPE=$2 +OLD=$3 +NEW=$4 declare -a KEYWORDS #KEYWORDS+=('\+\+\+\ ') -# URL access -KEYWORDS+=(URLConnection) -KEYWORDS+=(UrlConnectionDownloader) - -# Proxy settings -KEYWORDS+=(ProxySelector) - -# Android and java networking and 3rd party libs -KEYWORDS+=("openConnection\(") -KEYWORDS+=("java.net") -KEYWORDS+=("javax.net") -KEYWORDS+=(android.net) -KEYWORDS+=(android.webkit) - -# Third Party http libs -KEYWORDS+=(ch.boye.httpclientandroidlib.impl.client) -KEYWORDS+=(okhttp) - -# Intents -KEYWORDS+=(IntentHelper) -KEYWORDS+=(openUriExternal) -KEYWORDS+=(getHandlersForMimeType) -KEYWORDS+=(getHandlersForURL) -KEYWORDS+=(getHandlersForIntent) -# KEYOWRDS+=(android.content.Intent) # Common -KEYWORDS+=(startActivity) -KEYWORDS+=(startActivities) -KEYWORDS+=(startBroadcast) -KEYWORDS+=(sendBroadcast) -KEYWORDS+=(sendOrderedBroadcast) -KEYWORDS+=(startService) -KEYWORDS+=(bindService) -KEYWORDS+=(android.app.PendingIntent) -KEYWORDS+=(ActivityHandlerHelper.startIntentAndCatch) -KEYWORDS+=(AppLinksInterceptor) -KEYWORDS+=(AppLinksUseCases) - -# Rust symbols -KEYWORDS+=("connect\(") -KEYWORDS+=("recvmsg\(") -KEYWORDS+=("sendmsg\(") -KEYWORDS+=("::post\(") -KEYWORDS+=("::get\(") - -cd $REPO_DIR - -# Step 1: Generate match pattern based on in-scope keywords +initialize_java_symbols() { + # URL access + KEYWORDS+=(URLConnection) + KEYWORDS+=(UrlConnectionDownloader) + + # Proxy settings + KEYWORDS+=(ProxySelector) + + # Android and java networking and 3rd party libs + KEYWORDS+=("openConnection\(") + KEYWORDS+=("java.net") + KEYWORDS+=("javax.net") + KEYWORDS+=(android.net) + KEYWORDS+=(android.webkit) + + # Third Party http libs + KEYWORDS+=(ch.boye.httpclientandroidlib.impl.client) + KEYWORDS+=(okhttp) + + # Intents + KEYWORDS+=(IntentHelper) + KEYWORDS+=(openUriExternal) + KEYWORDS+=(getHandlersForMimeType) + KEYWORDS+=(getHandlersForURL) + KEYWORDS+=(getHandlersForIntent) + # KEYOWRDS+=(android.content.Intent) # Common + KEYWORDS+=(startActivity) + KEYWORDS+=(startActivities) + KEYWORDS+=(startBroadcast) + KEYWORDS+=(sendBroadcast) + KEYWORDS+=(sendOrderedBroadcast) + KEYWORDS+=(startService) + KEYWORDS+=(bindService) + KEYWORDS+=(android.app.PendingIntent) + KEYWORDS+=(ActivityHandlerHelper.startIntentAndCatch) + KEYWORDS+=(AppLinksInterceptor) + KEYWORDS+=(AppLinksUseCases) + KEYWORDS+=(ActivityDelegate) +} + +initialize_rust_symbols() { + KEYWORDS+=("connect\(") + KEYWORDS+=("recvmsg\(") + KEYWORDS+=("sendmsg\(") + KEYWORDS+=("::post\(") + KEYWORDS+=("::get\(") +} + +initialize_cpp_symbols() { + KEYWORDS+=("PR_GetHostByName") + KEYWORDS+=("PR_GetIPNodeByName") + KEYWORDS+=("PR_GetAddrInfoByName") + KEYWORDS+=("PR_StringToNetAddr") + + KEYWORDS+=("MDNS") + KEYWORDS+=("mDNS") + KEYWORDS+=("mdns") + + KEYWORDS+=("TRR") + KEYWORDS+=("trr") + + KEYWORDS+=("AsyncResolve") + KEYWORDS+=("asyncResolve") + KEYWORDS+=("ResolveHost") + KEYWORDS+=("resolveHost") + + KEYWORDS+=("SOCK_") + KEYWORDS+=("SOCKET_") + KEYWORDS+=("_SOCKET") + + KEYWORDS+=("UDPSocket") + KEYWORDS+=("TCPSocket") + + KEYWORDS+=("PR_Socket") + + KEYWORDS+=("SocketProvider") + KEYWORDS+=("udp-socket") + KEYWORDS+=("tcp-socket") + KEYWORDS+=("tcpsocket") + KEYWORDS+=("SOCKET") + KEYWORDS+=("mozilla.org/network") +} + +initialize_js_symbols() { + KEYWORDS+=("AsyncResolve\(") + KEYWORDS+=("asyncResolve\(") + KEYWORDS+=("ResolveHost\(") + KEYWORDS+=("resolveHost\(") + + KEYWORDS+=("udp-socket") + KEYWORDS+=("udpsocket") + KEYWORDS+=("tcp-socket") + KEYWORDS+=("tcpsocket") + KEYWORDS+=("SOCKET") + KEYWORDS+=("mozilla.org/network") +} + +# Step 1: Initialize scope of audit +EXT= +case "${SCOPE}" in + "java" | "kt" | "java-kt" ) + EXT="java kt" + SCOPE="java-kt" + initialize_java_symbols + ;; + "c-cpp" | "c-cxx" | "c" | "cxx" | "cpp" ) + EXT="c cpp h cxx hpp hxx" + SCOPE="c-cpp" + initialize_cpp_symbols + ;; + "rust" ) + EXT="rs" + initialize_rust_symbols + ;; + "js" ) + EXT="js jsm" + initialize_js_symbols + ;; + * ) + echo "requested language not recognized" + exit 1 + ;; +esac + +cd "$REPO_DIR" + +# Step 2: Generate match pattern based on in-scope keywords function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/$d}"; } -GREP_LINE="$(join_by \| ${KEYWORDS[@]})" +GREP_LINE="$(join_by \| "${KEYWORDS[@]}")" -# Step 2: Obtain patches for all in-scope files where a keyword is present -echo "Diffing patches-${OLD}-${NEW}-${SCOPE}.diff" -path= +# Step 3: Obtain patches for all in-scope files where a keyword is present +declare -a path for ext in ${EXT}; do - path="${path} *.${ext}" + path+=("*.${ext}") done +echo "Diffing patches-${OLD}-${NEW}-${SCOPE}.diff from all ${path[*]} files" # Exclude Deleted and Unmerged files from diff DIFF_FILTER=ACMRTXB -git diff --color=always --color-moved --diff-filter="${DIFF_FILTER}" -U20 -G"${GREP_LINE}" $OLD $NEW -- ${path} > patches-${OLD}-${NEW}-${SCOPE}.diff +git diff --color=always --color-moved --diff-filter="${DIFF_FILTER}" -U20 -G"${GREP_LINE}" "$OLD" "$NEW" -- "${path[@]}" > "patches-${OLD}-${NEW}-${SCOPE}.diff" -# Step 3: Highlight the keyword with an annoying, flashing color +# Step 4: Highlight the keyword with an annoying, flashing color export GREP_COLOR="05;37;41" # Capture the entire file and/or overlap with the previous match, add GREP_COLOR highlighting -egrep -A10000 -B10000 --color=always "${GREP_LINE}" patches-${OLD}-${NEW}-${SCOPE}.diff > keywords-$OLD-$NEW-$SCOPE.diff +grep -A10000 -B10000 --color=always -E "${GREP_LINE}" "patches-${OLD}-${NEW}-${SCOPE}.diff" > "keywords-$OLD-$NEW-$SCOPE.diff" # Add a 'XXX MATCH XXX' at the end of each matched line, easily searchable. -sed -i 's/$\x1b\[05;37;41.*$/\1 XXX MATCH XXX/' keywords-$OLD-$NEW-$SCOPE.diff +sed -i 's/$\x1b\[05;37;41.*$/\1 XXX MATCH XXX/' "keywords-$OLD-$NEW-$SCOPE.diff" -# Step 4: Review the code changes +# Step 5: Review the code changes echo "Diff generated. View it with:" echo " less -R $REPO_DIR/keywords-$OLD-$NEW-$SCOPE.diff"

1 0

[tor-browser-spec/master] Bug 40010: Use full patch diff for code audit
by gk＠torproject.org 19 Mar '21

19 Mar '21

commit 49b2b716aaa4fdc186202584fefccbc1ce62c479 Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Thu Jan 21 21:23:21 2021 +0000 Bug 40010: Use full patch diff for code audit --- audits/java_audit.sh | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/audits/java_audit.sh b/audits/java_audit.sh index 1417011..b1183eb 100644 --- a/audits/java_audit.sh +++ b/audits/java_audit.sh @@ -11,6 +11,7 @@ OLD=$2 NEW=$3 SCOPE="java" # string: this is the java audit +EXT="java kt" declare -a KEYWORDS @@ -62,21 +63,28 @@ KEYWORDS+=("::get\(") cd $REPO_DIR -if [ ! -f "release-${OLD}-${NEW}.diff" ]; -then - echo "Diffing release-${OLD}-${NEW}.diff" - git diff --color=always --color-moved $OLD $NEW -U20 > release-${OLD}-${NEW}.diff -fi - -echo "Done with diff" - +# Step 1: Generate match pattern based on in-scope keywords function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/$d}"; } GREP_LINE="$(join_by \| ${KEYWORDS[@]})" +# Step 2: Obtain patches for all in-scope files where a keyword is present +echo "Diffing patches-${OLD}-${NEW}-${SCOPE}.diff" +path= +for ext in ${EXT}; do + path="${path} *.${ext}" +done +# Exclude Deleted and Unmerged files from diff +DIFF_FILTER=ACMRTXB +git diff --color=always --color-moved --diff-filter="${DIFF_FILTER}" -U20 -G"${GREP_LINE}" $OLD $NEW -- ${path} > patches-${OLD}-${NEW}-${SCOPE}.diff + +# Step 3: Highlight the keyword with an annoying, flashing color export GREP_COLOR="05;37;41" +# Capture the entire file and/or overlap with the previous match, add GREP_COLOR highlighting +egrep -A10000 -B10000 --color=always "${GREP_LINE}" patches-${OLD}-${NEW}-${SCOPE}.diff > keywords-$OLD-$NEW-$SCOPE.diff -# XXX: Arg this sometimes misses file context -egrep -A40 -B40 --color=always "${GREP_LINE}" release-${OLD}-${NEW}.diff > keywords-${OLD}-${NEW}-$SCOPE.diff +# Add a 'XXX MATCH XXX' at the end of each matched line, easily searchable. +sed -i 's/$\x1b\[05;37;41.*$/\1 XXX MATCH XXX/' keywords-$OLD-$NEW-$SCOPE.diff +# Step 4: Review the code changes echo "Diff generated. View it with:" echo " less -R $REPO_DIR/keywords-$OLD-$NEW-$SCOPE.diff"

1 0

[tor-browser-spec/master] Rename java_audit.sh as code_audit.sh
by gk＠torproject.org 19 Mar '21

19 Mar '21

commit a93a6c96555ad0a8816cc08d9006794dc7b29e28 Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Tue Feb 2 00:34:57 2021 +0000 Rename java_audit.sh as code_audit.sh --- audits/{java_audit.sh => code_audit.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/audits/java_audit.sh b/audits/code_audit.sh similarity index 100% rename from audits/java_audit.sh rename to audits/code_audit.sh

1 0

[tor-browser-spec/master] Clean up java_audit script
by gk＠torproject.org 19 Mar '21

19 Mar '21

commit b5a9016a3d7ca0d6843a357f2ffc0118ad5bfc50 Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Thu Jan 21 21:11:55 2021 +0000 Clean up java_audit script --- audits/java_audit.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/audits/java_audit.sh b/audits/java_audit.sh index 51c9bf4..1417011 100644 --- a/audits/java_audit.sh +++ b/audits/java_audit.sh @@ -61,34 +61,22 @@ KEYWORDS+=("::post\(") KEYWORDS+=("::get\(") cd $REPO_DIR -#function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/ $d}"; } -#GREP_LINE="$(join_by \-G ${KEYWORDS[@]})" - -#base=`git merge-base ${OLD} ${NEW}` if [ ! -f "release-${OLD}-${NEW}.diff" ]; -#if [ ! -f "release-${base}-${NEW}.diff" ]; then echo "Diffing release-${OLD}-${NEW}.diff" - #echo "Diffing release-${base}-${NEW}.diff" git diff --color=always --color-moved $OLD $NEW -U20 > release-${OLD}-${NEW}.diff - #git diff --color=always --color-moved $base $NEW -U20 > release-${base}-${NEW}.diff - #git diff --color=always --color-moved -G${GREP_LINE} $OLD $NEW -U20 > release-${OLD}-${NEW}-G.diff fi echo "Done with diff" function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/$d}"; } - GREP_LINE="$(join_by \| ${KEYWORDS[@]})" -#GREP_LINE="\+\+\+ |$(join_by \| ${KEYWORDS[@]})" export GREP_COLOR="05;37;41" # XXX: Arg this sometimes misses file context -#egrep -A40 -B40 --color=always "${GREP_LINE}" release-${base}-${NEW}.diff > keywords-${base}-${NEW}-$SCOPE.diff egrep -A40 -B40 --color=always "${GREP_LINE}" release-${OLD}-${NEW}.diff > keywords-${OLD}-${NEW}-$SCOPE.diff echo "Diff generated. View it with:" -#echo " less -R $REPO_DIR/keywords-$base-$NEW-$SCOPE.diff" echo " less -R $REPO_DIR/keywords-$OLD-$NEW-$SCOPE.diff"

1 0

[tor-browser-spec/master] Merge remote-tracking branch 'gitlab/merge-requests/26'
by gk＠torproject.org 19 Mar '21

19 Mar '21

commit 2d8705e699f2bf7a00bc52ca6a38b1b5326b93e4 Merge: 82aa163 6735a8b Author: Georg Koppen <gk(a)torproject.org> Date: Fri Mar 19 21:04:59 2021 +0000 Merge remote-tracking branch 'gitlab/merge-requests/26' audits/NETWORK_AUDIT_RUBRIC | 93 ++++++------------------ audits/code_audit.sh | 168 ++++++++++++++++++++++++++++++++++++++++++++ audits/java_audit.sh | 94 ------------------------- 3 files changed, 190 insertions(+), 165 deletions(-)

1 0