Merge branch 'master' into fix/container

.circleci/config.yml

@@ -11,31 +11,28 @@ jobs:
 
   style:
     docker:
-      - image: registry.opensuse.org/opensuse/leap:15.3
+      - image: registry.opensuse.org/opensuse/tumbleweed
     steps:
       - checkout
       - run:
           command: |
-            zypper -n ar https://download.opensuse.org/repositories/devel:/languages:/python:/backports/openSUSE_Leap_15.3 py
-            zypper --gpg-auto-import-keys ref py
             zypper -n in --no-recommends make python3-black python3-importlib-metadata
       - checkout
       - run: make test_python_style
 
 
   integration:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-2004:202201-02
     steps:
       - checkout
-      - run: 
+      - run:
           command: |
             pyenv versions
-            pyenv global 3.7.0
-            git clone https://github.com/coryb/osht ../osht
+            pyenv global 3.10.2
             docker pull registry.opensuse.org/devel/openqa/ci/containers/serviced
             sudo chown -R 1000:1000 .
-            make test_docker PRIVILEGED_TESTS=1 OSHT_LOCATION=$(pwd)/../osht/osht.sh
+            make test_docker PRIVILEGED_TESTS=1
 
 workflows:
   version: 2.1

README.md

@@ -1,4 +1,5 @@
 # openqa-trigger-from-obs
+[![CircleCI](https://dl.circleci.com/status-badge/img/gh/os-autoinst/openqa-trigger-from-obs/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/os-autoinst/openqa-trigger-from-obs/tree/master)
 
 This project is aiming to simplify the integration between the Open
 Build Service (OBS) and openqa.opensuse.org (o3).
@@ -41,10 +42,10 @@ to one of those projects.
 ## Changes needed for existing projects
 
 Typical steps:
-1. Investigate the corresponding part in scriptgen.py and change according to
+1. Investigate the corresponding part in *scriptgen.py* and change according to
    new requirements.
 2. Re-generate scripts in test folders with `make test_regen_all`
-3. Update .before scripts in test folders with `make test_update_before_files`
+3. Update *.before* scripts in test folders with `make test_update_before_files`
 4. Run consistency tests with `make test`
 5. Review and acknowledge impact on projects e.g. `git difftool`
 The commands will highlight all affected projects and exact changes and side
@@ -60,26 +61,28 @@ commit affects them in the future.
 
 1. Find a project with similar settings, create a copy of its xml file and
    tweak changes as needed.
-2. Create folder t/obs/ProjectName and generate scripts using
+2. Create folder *t/obs/ProjectName* and generate scripts using
    `make test_regen_all`
 3. Examine the script for the first phase (read `Testability` section above)
-   t/obs/ProjectName/read_files.sh
+   *t/obs/ProjectName/read_files.sh*
 4. Use generated commands `rsync --list-only` to create files 
-t/obs/ProjectName/*.lst
-the same way they are created in read_files.sh
+   *t/obs/ProjectName/\*.lst* 
+   the same way they are created in *read_files.sh*
 5. Generate rsync and openqa commands based on these new *.lst files
-`make test_update_before_files`
+   `make test_update_before_files`
 6. Run consistency test `make test`
 7. Review and test bash commands generated in, which will be executed in
    production:
-t/obs/ProjectName/print_rsync_iso.before
-t/obs/ProjectName/print_rsync_repo.before
-t/obs/ProjectName/print_openqa.before
-8. Add to a new git commit the xml file and t/obs/ProjectName folder
+   * t/obs/ProjectName/print_rsync_iso.before
+   * t/obs/ProjectName/print_rsync_repo.before
+   * t/obs/ProjectName/print_openqa.before
+8. Add to a new git commit the xml file and *t/obs/ProjectName* folder
+```
 git checkout -b add_projectname
 git add xml/obs/ProjectName.xml t/obs/ProjectName
 git commit -m 'Add ProjectName'
 git push origin add_projectname
+```
 9. Create a pull request from add_projectname branch and make sure CI shows
    green outcome
 
@@ -102,6 +105,11 @@ python3 script/scriptgen.py Leap:15.2:ToTest
 bash script/rsync.sh Leap:15.2:ToTest
 ```
 
+## Further information
+
+Find some presentation slides on
+https://slides.com/andriinikitin/obs-to-openqa
+
 ## License
 
 This project is licensed under the MIT license, see LICENSE file for details.

profiles/apparmor.d/opt.openqa-trigger-from-obs.script.rsync.sh

@@ -15,6 +15,11 @@
   /opt/openqa-trigger-from-obs/*:*/*/.run*/* rw,
   /opt/openqa-trigger-from-obs/*:*/*/.run_last rw,
   /opt/openqa-trigger-from-obs/*:*/*/files*.lst rw,
+  /opt/openqa-trigger-from-obs/*:*/*/Media*.lst rw,
+  /opt/openqa-trigger-from-obs/*:*/*/.Media*.lst* rw,
+  /opt/openqa-trigger-from-obs/*:*/*/products* rw,
+  /opt/openqa-trigger-from-obs/*:*/*/.products* rw,
+
   /opt/openqa-trigger-from-obs/*:*/.run*/ rw,
   /opt/openqa-trigger-from-obs/*:*/.run*/* rw,
   /opt/openqa-trigger-from-obs/*:*/.run_last rw,
@@ -47,14 +52,15 @@
   /usr/bin/uniq ix,
   /usr/bin/wc ix,
   /var/lib/openqa/osc-plugin-factory/factory-package-news/factory-package-news.py rUx,
-  /usr/share/openqa/script/client rPx -> /opt/openqa-trigger-from-obs/script/rsync.sh//openqa_client,
-  /var/lib/openqa/share/factory/repo/** rw, # need write permission because sometimes bsdtar iso here
+  /usr/share/openqa/script/openqa-cli rPx -> /opt/openqa-trigger-from-obs/script/rsync.sh//openqa_cli,
+  /usr/bin/openqa-cli rPx -> /opt/openqa-trigger-from-obs/script/rsync.sh//openqa_cli,
+  /var/lib/openqa/share/factory/repo/** rwl, # need write and link permissions, because bsdtar may extract iso here
   /var/lib/openqa/share/factory/{iso,hdd,other}/** r,
   /{usr/bin,bin}/bash mrix,
   owner /proc/*/fd/* w,
 
 
-  profile openqa_client {
+  profile openqa_cli {
     #include <abstractions/base>
     #include <abstractions/nameservice>
     #include <abstractions/openssl>
@@ -68,8 +74,9 @@
     /etc/openqa/client.conf r,
     /opt/openqa-trigger-from-obs/*:*/*/.run*/openqa*.log w,
     /opt/openqa-trigger-from-obs/*:*/.run*/openqa*.log w,
+    /usr/bin/openqa-cli rix,
     /usr/share/openqa/lib/** r,
-    /usr/share/openqa/script/client rix,
+    /usr/share/openqa/script/openqa-cli rix,
     /var/lib/openqa/.config/openqa/client.conf r,
     /var/lib/openqa/share/factory/{iso,hdd,repo,other}/** r,
 
@@ -96,6 +103,11 @@
     /opt/openqa-trigger-from-obs/*:*/products* rw,
     /opt/openqa-trigger-from-obs/*:*/.products* rw,
 
+    /opt/openqa-trigger-from-obs/*:*/*/Media*.lst rw,
+    /opt/openqa-trigger-from-obs/*:*/*/.Media*.lst* rw,
+    /opt/openqa-trigger-from-obs/*:*/*/products* rw,
+    /opt/openqa-trigger-from-obs/*:*/*/.products* rw,
+
     /usr/bin/rsync mrix,
     /var/lib/openqa/share/factory/{iso,hdd,repo,other}/** rw,
 

script/cfg.py

@@ -35,8 +35,12 @@
 read_files_repo_link3 = '''cp __envdir/REPOLINK/files_iso*.lst __envsub/
 '''
 
-def rsync_fix_dest(distri, version, staging):
+def rsync_fix_dest(distri, version, staging, use_staging_patterns):
     if not staging: return ''
+    if use_staging_patterns: return '''
+    staging_pattern=$flavor
+    [ -z "${flavor_staging[$staging_pattern]}" ] || staging_pattern="${flavor_staging[$staging_pattern]}"
+    dest=${dest//$staging_pattern/Staging:__STAGING-$staging_pattern}'''
     if version != 'Factory' and len(staging) == 1: return '''dest=${dest//$flavor/Staging:__STAGING-Staging-$flavor}'''
     return '''dest=${dest//$flavor/Staging:__STAGING-$flavor}'''
 
@@ -52,19 +56,19 @@ def rsync_commands(checksum):
         echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/${iso_folder[$flavor]}*$src.sha256 /var/lib/openqa/factory/other/$dest.sha256"'''
     return res
 
-rsync_iso = lambda distri, version, archs, staging, checksum, repo0folder: '''
+rsync_iso = lambda distri, version, archs, staging, checksum, repo0folder, use_staging_patterns: '''
 archs=(ARCHITECTURS)
 
 for flavor in {FLAVORLIST,}; do
     for arch in "${archs[@]}"; do
         filter=$flavor
-        [[ ! -v flavor_filter[@] ]] || [ -z "${flavor_filter[$flavor]}" ] || filter=${flavor_filter[$flavor]}
+        [ -z "${flavor_filter[$flavor]}" ] || filter=${flavor_filter[$flavor]}
         [[ ${norsync_filter[$filter]} != 1 ]] || continue
         src=$(grep "$filter" __envsub/files_iso.lst | grep $arch | head -n 1)
         ''' + rsync_iso_fix_src(archs) + '''
         [ ! -z "$src" ] || continue
         dest=$src
-        ''' + rsync_fix_dest(distri, version, staging) + '''
+        ''' + rsync_fix_dest(distri, version, staging, use_staging_patterns) + '''
         asset_folder=other
         [[ ! $dest =~ \.iso$  ]] || asset_folder=iso
         [[ ! $dest =~ \.(qcow2|raw|vhd|vhdx|xz)$ ]] || asset_folder=hdd
@@ -108,7 +112,7 @@ def pre_rsync_repo(repos):
 [ ! -f __envsub/files_iso.lst ] || test -n "$buildid" || buildid=$(cat __envsub/files_iso.lst | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
 [ -z "__STAGING" ] || buildid=${buildid//Build/Build__STAGING.}
 [ ! -f __envsub/files_repo.lst ] || ! grep -q -- "-POOL-" __envsub/files_repo.lst || additional_repo_suffix=-POOL
-[ -n "$buildid" ] || buildid=$(grep -hEo 'Build[0-9]+' __envsub/Media1_*.lst 2>/dev/null | head -n 1)
+[ -n "$buildid" ] || buildid=$(grep -hEo 'Build[0-9]+(.[0-9]+)?' __envsub/Media1_*.lst 2>/dev/null | head -n 1)
 
 for repo in {REPOOWNLIST,}; do
     while read src; do
@@ -132,24 +136,33 @@ def pre_rsync_repo(repos):
 
 rsync_repodir1 = '''
 archs=(ARCHITECTURREPO)
-[ ! -f __envsub/files_iso.lst ] || buildid=$(cat __envsub/files_iso.lst | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
-[ -n "$buildid" ] || buildid=$(grep -hEo 'Build[0-9]+' __envsub/Media1_*.lst 2>/dev/null | head -n 1)
+
 
 for arch in "${archs[@]}"; do
+    [ ! -f __envsub/files_iso.lst ] || buildid=$(cat __envsub/files_iso.lst | grep $arch | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
+    [ -n "$buildid" ] || buildid=$(grep -hEo 'Build[0-9]+(.[0-9]+)?' __envsub/Media1_*.lst 2>/dev/null | head -n 1)
+    if [[ ! -n $build ]] && [[ ! $buildid =~ (Build|Snapshot)[0-9]*[0-9]$ ]]; then
+        buildid=${buildid%%\.[[:alpha:]]*}
+    fi
+
     while read src; do
         [ ! -z "$src" ] || continue
         dest=$src
+        dest=${dest%-Build*}
         destPrefix=${dest%$arch*}
         destSuffix=${dest#$destPrefix}
         mid=''
-        dest=$destPrefix$mid$destSuffix'''
+        dest=$destPrefix$mid$destSuffix
+        [[ ! $src =~ .*\.license ]] || [[ $dest == *license* ]] || dest=$dest.license'''
 
 def rsync_repodir2():
     return '''
         dest=${dest//-Media2/}
-        echo rsync --timeout=3600 -rtlp4 --delete --specials RSYNCFILTER PRODUCTREPOPATH/*Media2/*  /var/lib/openqa/factory/repo/$dest-CURRENT-debuginfo/
+        Mdia=Media2
+        [[ ! $src =~ .*\.license ]] || Mdia=Media2.license
+        echo rsync --timeout=3600 -rtlp4 --delete --specials RSYNCFILTER PRODUCTREPOPATH/*$Mdia/  /var/lib/openqa/factory/repo/$dest-CURRENT-debuginfo/
         echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/$dest-CURRENT-debuginfo/ /var/lib/openqa/factory/repo/$dest-CURRENT-debuginfo/ /var/lib/openqa/factory/repo/$dest-$buildid-debuginfo
-    done < <(grep ${arch//i686/i586} __envsub/files_repo.lst | grep Media2)
+    done < <(grep ${arch//i686/i586} __envsub/files_repo.lst | grep Media2 )
 done
 '''
 
@@ -161,7 +174,9 @@ def rsync_repodir2():
     while read src; do
         [ ! -z "$src" ] || continue
         [[ ''' + dest + ''' =~ $arch ]] || [[ ''' + dest + ''' =~ ${arch//i686/i586} ]] || [[ "ARCHITECTURREPO" == . ]] || break
-        dest=''' + dest
+        dest=''' + dest + '''
+        [[ ! $src =~ .*\.license ]] || dest=$dest.license'''
+
 
 
 def openqa_call_fix_destiso(distri, version, staging):
@@ -211,8 +226,10 @@ def pre_openqa_call_start(repos):
     for arch in "${archs[@]}"; do
         filter=$flavor
         ''' + openqa_call_start_distri(flavor_distri) + '''
-        [[ ! -v flavor_filter[@] ]] || [ -z "${flavor_filter[$flavor]}" ] || filter=${flavor_filter[$flavor]}
+        [ -z "${flavor_filter[$flavor]}" ] || filter=${flavor_filter[$flavor]}
         [ $filter != Appliance ] || filter="qcow2"
+        version=VERSIONVALUE
+        if [ -z "${norsync_filter[$flavor]}" ] || [ -z $build1 ]; then {
         iso=$(grep "$filter" __envsub/files_iso.lst 2>/dev/null | grep $arch | head -n 1)
         ''' + openqa_call_start_fix_iso(archs) + '''
         build=$(echo $iso | grep -o -E '(Build|Snapshot)[^-]*' | grep -o -E '[0-9]\.?[0-9]+(\.[0-9]+)*' | tail -n 1) || :
@@ -226,14 +243,15 @@ def pre_openqa_call_start(repos):
         buildex=${buildex/.qcow2/}
         build1=$build
         destiso=$iso
-        version=VERSIONVALUE
         [ -z "__STAGING" ] || build1=__STAGING.$build
         ''' + openqa_call_fix_destiso(distri, version, staging) + '''
         repo0folder=${destiso%.iso}
         ''' + (repo0folder if repo0folder else "") + '''
         [ "$arch" != . ] || arch=x86_64
         ''' + openqa_call_news(news, news_archs) + '''
-        echo "/usr/share/openqa/script/client isos post --host localhost \\\\\"
+        }
+        fi
+        echo "/usr/bin/openqa-cli api -X post isos?async=1 \\\\\"
 (
  echo \" DISTRI=$distri \\\\
  ARCH=$arch \\\\
@@ -266,7 +284,7 @@ def openqa_call_start_ex(checksum):
    :
  elif [[ $destiso =~ \.iso$ ]]; then
    echo \" ''' + openqa_call_start_ex1(checksum, 'ISO')  + '''\"
- elif [[ $destiso =~ \.(hdd|qcow2|raw\.xz|raw\.gz|vhdx\.xz|vmdk\.xz)$ ]]; then
+ elif [[ $destiso =~ \.(hdd|qcow2|raw|raw\.xz|raw\.gz|vhdx\.xz|vmdk\.xz)$ ]]; then
    echo \" ''' + openqa_call_start_ex1(checksum, 'HDD_1')  + '''\"
  elif [ -n "$destiso" ]; then
    echo \" ''' + openqa_call_start_ex1(checksum, 'ASSET_1')  + '''\"
@@ -372,13 +390,18 @@ def openqa_call_repot1_debugsource():
 openqa_call_repot1 = lambda: '''
         while read src; do
             dest=$src
+            dest=${dest%-Build*}
+            dest=${dest%.license}
             destPrefix=${dest%$arch*}
             destSuffix=${dest#$destPrefix}
             mid=''
             dest=$destPrefix$mid$destSuffix
             repoPrefix=${dest%-Media*}
             repoSuffix=${dest#$repoPrefix}
-            dest=$repoPrefix-Build$build$repoSuffix
+            [[ ! $src =~ .license ]] || repoPrefix=$repoPrefix.license
+            B=$(grep -oE "Build|Snapshot" __envsub/files_iso.lst 2>/dev/null | head -1)
+            [ ! -z "$B" ] || B=Build
+            dest=$repoPrefix-$B$build$repoSuffix
             repoKey=REPOKEY
             repoKey=${repoKey^^}
             repoKey=${repoKey//-/_}
@@ -400,7 +423,9 @@ def openqa_call_repot1_debugsource():
 openqa_call_repot1_dest = lambda dest: '''
         while read src; do
             [[ ''' + dest + ''' =~ $arch ]] || [[ ''' + dest + ''' =~ ${arch//i686/i586} ]] || [[ "ARCHITECTURREPO" == . ]] || break
-            dest=''' + dest + '''-$buildex
+            dest=''' + dest + '''
+            [[ ! $src =~ .*\.license ]] || dest=$dest.license
+            dest=$dest-$buildex
             repoKey=REPOKEY
             ''' + openqa_call_repot1_debugsource() + '''
             repoKey=${repoKey^^}

script/rsync.sh

@@ -36,7 +36,7 @@ set +e
     [ -e "$subfolder/print_openqa.sh" ] || { >&2 echo "No file found: {$subfolder/print_openqa.sh}"; exit 1; }
 
     # nowhere to log yet as we haven't created $logfolder
-    bash -e "$subfolder/read_files.sh"
+    bash -e "$subfolder/read_files.sh" ||  { >&2 echo "read_files.sh failed for $subfolder in enviroment $environ"; exit 1; }
 
     if [ -e $subfolder/.run_last ] && [ -z "$(diff --brief $subfolder $subfolder/.run_last | grep '.lst')" ]; then
         >&2 echo "No changes found since last run, skipping {$subfolder}"
@@ -47,12 +47,13 @@ set +e
         if [[ "$environ" != *Factory* ]] && [[ "$environ" != *MicroOS* ]]; then
             builds="$(grep -h -o -E 'Build[0-9](\.|[0-9]+)*[0-9]+' $subfolder/*.lst 2>/dev/null)" || :
         else
-            builds="$(grep -h -o -E '20[0-9]{6}' $subfolder/*.lst 2>/dev/null)" || :
+            # for *Factory* we include base branch for checking Snapshot consistency
+            builds="$(grep -h -o -E '20[0-9]{6}' $environ/base/*.lst 2>/dev/null || : ; grep -h -o -E '20[0-9]{6}' $subfolder/*.lst 2>/dev/null)" || :
         fi
 
         if [ -n "$builds" ] && [ $(echo "$builds" | sort | uniq | wc -l) -gt 1 ]; then
-            >&2 echo "Conflicting builds found {$builds}, skipping {$subfolder}"
-            continue
+            >&2 echo "Conflicting builds found {$builds}, exiting because of conflict in {$subfolder}"
+            exit 0
         fi
     fi