[{B,D,E,F}/master-next,1/2] UBUNTU: [Packaging] dkms -- try launchpad librarian for pool downloads
diff mbox series

Message ID 20191101214822.76610-2-apw@canonical.com
State New
Headers show
Series
  • [{B,D,E,F}/master-next,1/2] UBUNTU: [Packaging] dkms -- try launchpad librarian for pool downloads
Related show

Commit Message

Andy Whitcroft Nov. 1, 2019, 9:48 p.m. UTC
When we build a kernel we capture the current dkms package versions.
This allows us to keep the versions syncronised in an entire SRU cycle
between the master and derivative kernels.  It also allows us to maintain
the same versions on respin.  This suffers from the issue that they may
expire from the archive pool if the dkms packages are revved.

The same packages are available in the launchpad librarian and indeed
(now) via well known names.  So when are attempting to download from the
archive pool first attempt to get the package from the primary archive
librarian first.  If this fails continue to do the pool scan as normal.
This increases the chance we will find an appropriate package regardless
of the currently published packages, whilst retaining the ability to build
in a more restricted environment (such as when we have no internet access).

BugLink: http://bugs.launchpad.net/bugs/1850958
Signed-off-by: Andy Whitcroft <apw@canonical.com>
---
 debian/scripts/dkms-build | 98 ++++++++++++++++++++++++---------------
 1 file changed, 60 insertions(+), 38 deletions(-)

Comments

Marcelo Henrique Cerri Nov. 4, 2019, 4:48 p.m. UTC | #1
On Fri, Nov 01, 2019 at 09:48:21PM +0000, Andy Whitcroft wrote:
> When we build a kernel we capture the current dkms package versions.
> This allows us to keep the versions syncronised in an entire SRU cycle
> between the master and derivative kernels.  It also allows us to maintain
> the same versions on respin.  This suffers from the issue that they may
> expire from the archive pool if the dkms packages are revved.
> 
> The same packages are available in the launchpad librarian and indeed
> (now) via well known names.  So when are attempting to download from the
> archive pool first attempt to get the package from the primary archive
> librarian first.  If this fails continue to do the pool scan as normal.
> This increases the chance we will find an appropriate package regardless
> of the currently published packages, whilst retaining the ability to build
> in a more restricted environment (such as when we have no internet access).
> 
> BugLink: http://bugs.launchpad.net/bugs/1850958
> Signed-off-by: Andy Whitcroft <apw@canonical.com>
> ---
>  debian/scripts/dkms-build | 98 ++++++++++++++++++++++++---------------
>  1 file changed, 60 insertions(+), 38 deletions(-)
> 
> diff --git a/debian/scripts/dkms-build b/debian/scripts/dkms-build
> index 08c1a8fa1bd4..3c3c28f75c43 100755
> --- a/debian/scripts/dkms-build
> +++ b/debian/scripts/dkms-build
> @@ -26,56 +26,78 @@ built_using_record()
>  	sed -i -e "s/^\(linux:BuiltUsing=.*\)/\1$built_using, /" "$subst"
>  }
>  
> +# ABI: returns present in $? and located path in lpackage_path when found.
> +package_present()
> +{
> +	for lpackage_path in "$1"_*.deb

In general it looks good. I would make this variable local and use the
opportunity to cleanup the shellcheck errors.


> +	do
> +		break
> +	done
> +	[ -f "$lpackage_path" ]
> +}
> +
>  # Download and extract the DKMS package -- note there may be more
>  # than one package to install.
>  for package_path in "$@"
>  do
> -	echo "II: dkms-build downloading $package ($(basename $package_path))"
> +	package_file=$(basename "$package_path")
> +	echo "II: dkms-build downloading $package ($package_file)"
>  	rpackage=$( echo "$package_path" | sed -e 's@.*/@@' -e 's@_.*@@' )
>  	lpackage=$( echo "$rpackage" | sed -e 's@=.*@@' )
>  
> -	case "$package_path" in
> -	pool/*)
> -		for pool in $( apt-cache policy | grep '^ [^ ]' | sort -r -n -k 1,1 -s | \
> -		    awk '
> -			($2 ~ /^http/) {
> -			    if (!($2 in E)) {
> -				E[$2]=1;
> -				print $2;
> -			    }
> -			}
> -		    ')
> -		do
> -			for lpackage_path in "$lpackage"_*.deb
> -			do
> -				break
> -			done
> -			if [ -f "$lpackage_path" ]; then
> +	while true
> +	do
> +		if package_present "$lpackage"; then
> +			break
> +		fi
> +		case "$package_path" in
> +		pool/*)
> +			# Attempt download from the launchpad librarian first.
> +			wget "https://launchpad.net/ubuntu/+archive/primary/+files/$package_file" || true
> +			if package_present "$lpackage"; then
>  				break
>  			fi
> -			url="$pool/$package_path"
> -			wget "$url" && break || true
> -			# No components in PPAs.
> -			url=$(echo "$url" | sed -e 's@/pool/[^/]*/@/pool/main/@')
> -			wget "$url" && break || true
> -		done
> -		;;
> -	http*:*)
> -		wget "$package_path"
> -		;;
> -	*/*)
> -		cp -p "$package_path" .
> -		;;
> -	*)
> -		apt-get download "$rpackage"
> -		;;
> -	esac
> -	dpkg -x "$lpackage"_*.deb "$package"
>  
> -	for lpackage_path in "$lpackage"_*.deb
> -	do
> +			# Download from the available pools.
> +			for pool in $( apt-cache policy | grep '^ [^ ]' | sort -r -n -k 1,1 -s | \
> +			    awk '
> +				($2 ~ /^http/) {
> +				    if (!($2 in E)) {
> +					E[$2]=1;
> +					print $2;
> +				    }
> +				}
> +			    ')
> +			do
> +				if package_present "$lpackage"; then
> +					break
> +				fi
> +				url="$pool/$package_path"
> +				wget "$url" && break || true
> +				# No components in PPAs.
> +				url=$(echo "$url" | sed -e 's@/pool/[^/]*/@/pool/main/@')
> +				wget "$url" && break || true
> +			done
> +			;;
> +		http*:*)
> +			wget "$package_path"
> +			;;
> +		*/*)
> +			cp -p "$package_path" .
> +			;;
> +		*)
> +			apt-get download "$rpackage"
> +			;;
> +		esac
>  		break
>  	done
> +	if ! package_present "$lpackage"; then
> +		echo "EE: $lpackage not found"
> +		exit 1
> +	fi
> +
> +	dpkg -x "$lpackage"_*.deb "$package"
> +
>  	lversion=$( echo "$lpackage_path" | sed -e 's@.*/@@' -e 's@_[^_]*$@@' -e 's@.*_@@')
>  	built_using_record "$srcdir/debian/$pkgname.substvars" "$built_using$lpackage (= $lversion)"
>  done
> -- 
> 2.20.1
> 
> 
> -- 
> kernel-team mailing list
> kernel-team@lists.ubuntu.com
> https://lists.ubuntu.com/mailman/listinfo/kernel-team
Andy Whitcroft Nov. 6, 2019, 5:22 p.m. UTC | #2
On Mon, Nov 04, 2019 at 01:48:36PM -0300, Marcelo Henrique Cerri wrote:
> On Fri, Nov 01, 2019 at 09:48:21PM +0000, Andy Whitcroft wrote:
> > When we build a kernel we capture the current dkms package versions.
> > This allows us to keep the versions syncronised in an entire SRU cycle
> > between the master and derivative kernels.  It also allows us to maintain
> > the same versions on respin.  This suffers from the issue that they may
> > expire from the archive pool if the dkms packages are revved.
> > 
> > The same packages are available in the launchpad librarian and indeed
> > (now) via well known names.  So when are attempting to download from the
> > archive pool first attempt to get the package from the primary archive
> > librarian first.  If this fails continue to do the pool scan as normal.
> > This increases the chance we will find an appropriate package regardless
> > of the currently published packages, whilst retaining the ability to build
> > in a more restricted environment (such as when we have no internet access).
> > 
> > BugLink: http://bugs.launchpad.net/bugs/1850958
> > Signed-off-by: Andy Whitcroft <apw@canonical.com>
> > ---
> >  debian/scripts/dkms-build | 98 ++++++++++++++++++++++++---------------
> >  1 file changed, 60 insertions(+), 38 deletions(-)
> > 
> > diff --git a/debian/scripts/dkms-build b/debian/scripts/dkms-build
> > index 08c1a8fa1bd4..3c3c28f75c43 100755
> > --- a/debian/scripts/dkms-build
> > +++ b/debian/scripts/dkms-build
> > @@ -26,56 +26,78 @@ built_using_record()
> >  	sed -i -e "s/^\(linux:BuiltUsing=.*\)/\1$built_using, /" "$subst"
> >  }
> >  
> > +# ABI: returns present in $? and located path in lpackage_path when found.
> > +package_present()
> > +{
> > +	for lpackage_path in "$1"_*.deb
> 
> In general it looks good. I would make this variable local and use the
> opportunity to cleanup the shellcheck errors.

The variable is documented as one of the return values for this
function.  Making it local would break that.

I am trying to do as few changes as possible to the script in this
update.

-apw

Patch
diff mbox series

diff --git a/debian/scripts/dkms-build b/debian/scripts/dkms-build
index 08c1a8fa1bd4..3c3c28f75c43 100755
--- a/debian/scripts/dkms-build
+++ b/debian/scripts/dkms-build
@@ -26,56 +26,78 @@  built_using_record()
 	sed -i -e "s/^\(linux:BuiltUsing=.*\)/\1$built_using, /" "$subst"
 }
 
+# ABI: returns present in $? and located path in lpackage_path when found.
+package_present()
+{
+	for lpackage_path in "$1"_*.deb
+	do
+		break
+	done
+	[ -f "$lpackage_path" ]
+}
+
 # Download and extract the DKMS package -- note there may be more
 # than one package to install.
 for package_path in "$@"
 do
-	echo "II: dkms-build downloading $package ($(basename $package_path))"
+	package_file=$(basename "$package_path")
+	echo "II: dkms-build downloading $package ($package_file)"
 	rpackage=$( echo "$package_path" | sed -e 's@.*/@@' -e 's@_.*@@' )
 	lpackage=$( echo "$rpackage" | sed -e 's@=.*@@' )
 
-	case "$package_path" in
-	pool/*)
-		for pool in $( apt-cache policy | grep '^ [^ ]' | sort -r -n -k 1,1 -s | \
-		    awk '
-			($2 ~ /^http/) {
-			    if (!($2 in E)) {
-				E[$2]=1;
-				print $2;
-			    }
-			}
-		    ')
-		do
-			for lpackage_path in "$lpackage"_*.deb
-			do
-				break
-			done
-			if [ -f "$lpackage_path" ]; then
+	while true
+	do
+		if package_present "$lpackage"; then
+			break
+		fi
+		case "$package_path" in
+		pool/*)
+			# Attempt download from the launchpad librarian first.
+			wget "https://launchpad.net/ubuntu/+archive/primary/+files/$package_file" || true
+			if package_present "$lpackage"; then
 				break
 			fi
-			url="$pool/$package_path"
-			wget "$url" && break || true
-			# No components in PPAs.
-			url=$(echo "$url" | sed -e 's@/pool/[^/]*/@/pool/main/@')
-			wget "$url" && break || true
-		done
-		;;
-	http*:*)
-		wget "$package_path"
-		;;
-	*/*)
-		cp -p "$package_path" .
-		;;
-	*)
-		apt-get download "$rpackage"
-		;;
-	esac
-	dpkg -x "$lpackage"_*.deb "$package"
 
-	for lpackage_path in "$lpackage"_*.deb
-	do
+			# Download from the available pools.
+			for pool in $( apt-cache policy | grep '^ [^ ]' | sort -r -n -k 1,1 -s | \
+			    awk '
+				($2 ~ /^http/) {
+				    if (!($2 in E)) {
+					E[$2]=1;
+					print $2;
+				    }
+				}
+			    ')
+			do
+				if package_present "$lpackage"; then
+					break
+				fi
+				url="$pool/$package_path"
+				wget "$url" && break || true
+				# No components in PPAs.
+				url=$(echo "$url" | sed -e 's@/pool/[^/]*/@/pool/main/@')
+				wget "$url" && break || true
+			done
+			;;
+		http*:*)
+			wget "$package_path"
+			;;
+		*/*)
+			cp -p "$package_path" .
+			;;
+		*)
+			apt-get download "$rpackage"
+			;;
+		esac
 		break
 	done
+	if ! package_present "$lpackage"; then
+		echo "EE: $lpackage not found"
+		exit 1
+	fi
+
+	dpkg -x "$lpackage"_*.deb "$package"
+
 	lversion=$( echo "$lpackage_path" | sed -e 's@.*/@@' -e 's@_[^_]*$@@' -e 's@.*_@@')
 	built_using_record "$srcdir/debian/$pkgname.substvars" "$built_using$lpackage (= $lversion)"
 done