[OpenWrt-Devel,V3,2/2] script/feeds: add a new command that allows generating a new feeds.conf
diff mbox series

Message ID 20190605063409.27769-1-john@phrozen.org
State New
Delegated to: John Crispin
Headers show
Series
  • Untitled series #111881
Related show

Commit Message

John Crispin June 5, 2019, 6:34 a.m. UTC
This can be used inside build setups for easy feeds.conf generation.

Signed-off-by: John Crispin <john@phrozen.org>
---
 scripts/feeds | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

Comments

Karl Palsson June 5, 2019, 10:17 a.m. UTC | #1
John Crispin <john@phrozen.org> wrote:
> This can be used inside build setups for easy feeds.conf
> generation.


Could you give us an example of how this is actually easy, or
what sort of functionality this is providing beyond "cat
feeds.conf.default feeds.conf.extra > feeds.conf"

It seems like a lot of perl for a narrow usecase.

Sincerely,
Karl Palsson


> 
> Signed-off-by: John Crispin <john@phrozen.org>
> ---
>  scripts/feeds | 42 ++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 42 insertions(+)
> 
> diff --git a/scripts/feeds b/scripts/feeds
> index 304ef6cbaf..7cd4639ca6 100755
> --- a/scripts/feeds
> +++ b/scripts/feeds
> @@ -7,6 +7,7 @@ use metadata;
>  use warnings;
>  use strict;
>  use Cwd 'abs_path';
> +use File::Copy;
>  
>  chdir "$FindBin::Bin/..";
>  $ENV{TOPDIR} //= getcwd();
> @@ -819,6 +820,42 @@ sub update {
>  	return $failed;
>  }
>  
> +sub setup {
> +	my %opts;
> +
> +	getopts('bh', \%opts);
> +
> +	if ($opts{h}) {
> +		usage();
> +		return 0;
> +	}
> +
> +	if ($opts{b}) {
> +		copy("feeds.conf.default", "feeds.conf") or die "Copy failed: $!"
> +	} else {
> +		unlink "feeds.conf"
> +	}
> +
> +	open(my $fd, ">>feeds.conf");
> +	while (my $entry = shift @ARGV) {
> +		my ($type, $name, $src) = split /,/, $entry;
> +
> +		$update_method{$type} or do {
> +			warn "Unknown type '$type' in parameter $entry\n";
> +			unlink "feeds.conf";
> +			return 1;
> +		};
> +		if ($name =~ /\s/ || $src =~ /\s/) {
> +			warn "Feed names or sources may not contain whitespace characters in parameter $entry\n";
> +			unlink "feeds.conf";
> +			return 1;
> +		}
> +		printf $fd "%s %s %s\n", $type, $name, $src;
> +	}
> +
> +	return 0;
> +}
> +
>  sub feed_config() {
>  	foreach my $feed (@feeds) {
>  		my $installed = (-f "feeds/$feed->[1].index");
> @@ -870,6 +907,10 @@ Commands:
>  	    -i :           Recreate the index only. No feed update from repository is performed.
>  	    -f :           Force updating feeds even if there are changed, uncommitted files.
>  
> +	setup [options] <type,name,link> <type,name,link> ...: generate feeds.conf
> +	Options:
> +	    -b :           Use feeds.conf.default as base for new feeds.conf.
> +
>  	clean:             Remove downloaded/generated files.
>  
>  EOF
> @@ -883,6 +924,7 @@ my %commands = (
>  	'search' => \&search,
>  	'uninstall' => \&uninstall,
>  	'feed_config' => \&feed_config,
> +	'setup' => \&setup,
>  	'clean' => sub {
>  		system("rm -rf ./feeds ./package/feeds");
>  	}
> -- 
> 2.20.1
> 
> 
> _______________________________________________
> openwrt-devel mailing list
> openwrt-devel@lists.openwrt.org
> https://lists.openwrt.org/mailman/listinfo/openwrt-devel
John Crispin June 5, 2019, 10:30 a.m. UTC | #2
On 05/06/2019 12:17, Karl Palsson wrote:
> John Crispin <john@phrozen.org> wrote:
>> This can be used inside build setups for easy feeds.conf
>> generation.
>
> Could you give us an example of how this is actually easy, or
> what sort of functionality this is providing beyond "cat
> feeds.conf.default feeds.conf.extra > feeds.conf"
>
> It seems like a lot of perl for a narrow usecase.
>
> Sincerely,
> Karl Palsson

This was brought up as a missing feature by the prpl folks. I considered 
on how to best implement this and find that having proper tooling is 
much better than having to carry around an extra file that is cat. being 
able to build the feeds.conf dynamically like this just seems much 
cleaner to me and will allow downstream users, vendors, odms and 
integrators to have less need to patch their trees to death.

     John


>
>> Signed-off-by: John Crispin <john@phrozen.org>
>> ---
>>   scripts/feeds | 42 ++++++++++++++++++++++++++++++++++++++++++
>>   1 file changed, 42 insertions(+)
>>
>> diff --git a/scripts/feeds b/scripts/feeds
>> index 304ef6cbaf..7cd4639ca6 100755
>> --- a/scripts/feeds
>> +++ b/scripts/feeds
>> @@ -7,6 +7,7 @@ use metadata;
>>   use warnings;
>>   use strict;
>>   use Cwd 'abs_path';
>> +use File::Copy;
>>   
>>   chdir "$FindBin::Bin/..";
>>   $ENV{TOPDIR} //= getcwd();
>> @@ -819,6 +820,42 @@ sub update {
>>   	return $failed;
>>   }
>>   
>> +sub setup {
>> +	my %opts;
>> +
>> +	getopts('bh', \%opts);
>> +
>> +	if ($opts{h}) {
>> +		usage();
>> +		return 0;
>> +	}
>> +
>> +	if ($opts{b}) {
>> +		copy("feeds.conf.default", "feeds.conf") or die "Copy failed: $!"
>> +	} else {
>> +		unlink "feeds.conf"
>> +	}
>> +
>> +	open(my $fd, ">>feeds.conf");
>> +	while (my $entry = shift @ARGV) {
>> +		my ($type, $name, $src) = split /,/, $entry;
>> +
>> +		$update_method{$type} or do {
>> +			warn "Unknown type '$type' in parameter $entry\n";
>> +			unlink "feeds.conf";
>> +			return 1;
>> +		};
>> +		if ($name =~ /\s/ || $src =~ /\s/) {
>> +			warn "Feed names or sources may not contain whitespace characters in parameter $entry\n";
>> +			unlink "feeds.conf";
>> +			return 1;
>> +		}
>> +		printf $fd "%s %s %s\n", $type, $name, $src;
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>>   sub feed_config() {
>>   	foreach my $feed (@feeds) {
>>   		my $installed = (-f "feeds/$feed->[1].index");
>> @@ -870,6 +907,10 @@ Commands:
>>   	    -i :           Recreate the index only. No feed update from repository is performed.
>>   	    -f :           Force updating feeds even if there are changed, uncommitted files.
>>   
>> +	setup [options] <type,name,link> <type,name,link> ...: generate feeds.conf
>> +	Options:
>> +	    -b :           Use feeds.conf.default as base for new feeds.conf.
>> +
>>   	clean:             Remove downloaded/generated files.
>>   
>>   EOF
>> @@ -883,6 +924,7 @@ my %commands = (
>>   	'search' => \&search,
>>   	'uninstall' => \&uninstall,
>>   	'feed_config' => \&feed_config,
>> +	'setup' => \&setup,
>>   	'clean' => sub {
>>   		system("rm -rf ./feeds ./package/feeds");
>>   	}
>> -- 
>> 2.20.1
>>
>>
>> _______________________________________________
>> openwrt-devel mailing list
>> openwrt-devel@lists.openwrt.org
>> https://lists.openwrt.org/mailman/listinfo/openwrt-devel
>>
>> _______________________________________________
>> openwrt-devel mailing list
>> openwrt-devel@lists.openwrt.org
>> https://lists.openwrt.org/mailman/listinfo/openwrt-devel
Petr Štetiar June 5, 2019, 10:45 a.m. UTC | #3
John Crispin <john@phrozen.org> [2019-06-05 12:30:57]:

> On 05/06/2019 12:17, Karl Palsson wrote:
> > It seems like a lot of perl for a narrow usecase.

It seems like a good starting point, but it makes me wonder who is going to
touch this afterwards due to the Perl :-)

> This was brought up as a missing feature by the prpl folks. I considered on
> how to best implement this and find that having proper tooling is much
> better than having to carry around an extra file that is cat. being able to
> build the feeds.conf dynamically like this just seems much cleaner to me and
> will allow downstream users, vendors, odms and integrators to have less need
> to patch their trees to death.

BTW been using following for ages:

 cat scripts/update-feeds.sh 
 #!/bin/bash

 for feed in $(ls -1 feeds | grep -v '\.'); do	
	pushd feeds/$feed > /dev/null

	remote=$(git ls-remote --get-url)
	comment=$(git log -1 --pretty=format:"%h: %s")
	url=$(git log -1 --pretty=format:"src-git $feed ${remote}^%h")

	popd > /dev/null

	echo "# $comment" >> feeds.conf
	echo $url >> feeds.conf
 done

producing following:

 # 21b29f3faf8b: Merge pull request #2513 from musashino205/l10n/base-upd-ja
 src-git luci git://git.openwrt.org/project/luci.git^21b29f3faf8b
 # e4ab7b4fec30: znc: fix patches applying
 src-git packages https://github.com/openwrt/packages.git^e4ab7b4fec30

Planned to convert it into make target one day...

-- ynezz
Karl Palsson June 5, 2019, 11:35 a.m. UTC | #4
John Crispin <john@phrozen.org> wrote:
> 
> On 05/06/2019 12:17, Karl Palsson wrote:
> > John Crispin <john@phrozen.org> wrote:
> >> This can be used inside build setups for easy feeds.conf
> >> generation.
> >
> > Could you give us an example of how this is actually easy, or
> > what sort of functionality this is providing beyond "cat
> > feeds.conf.default feeds.conf.extra > feeds.conf"
> >
> > It seems like a lot of perl for a narrow usecase.
> >
> > Sincerely,
> > Karl Palsson
> 
> This was brought up as a missing feature by the prpl folks. I
> considered on how to best implement this and find that having
> proper tooling is much better than having to carry around an
> extra file that is cat. being able to build the feeds.conf
> dynamically like this just seems much cleaner to me and will
> allow downstream users, vendors, odms and integrators to have
> less need to patch their trees to death.

So, they still have to have a script, but now the script has...


...
./scripts/feeds setup -b src-git,private-aa,git://blah
src-link,private-bb,/wop/blah
...

instead of
...
cp feeds.conf.default feeds.conf
echo "src-git private-aa git://blah" >> feeds.conf
echo "src-link private-bb /wop/blah" >> feeds.conf
...

I mean, _yes_ it's "simpler" but it's only simpler by bringing in
new tools with new layers of abstraction. I really question
whether that's actually simpler for anyone in the long run, and
also how this really counts as a "missing feature" There's still
going to be a requirement for that vendor script.

Sincerely,
Karl Palsson
Bjørn Mork June 5, 2019, 12:04 p.m. UTC | #5
A little late into this discussion, but something like the attached
patch would suite my personal usage patterns much better.  I don't
really need to update or change my local additions to feeds.conf at all.
I just want to add them to the defaults for whatever version I am
currently building.

Another alternative would be to make scripts/feeds look for an optional
feeds.conf.local when falling back to feeds.conf.default.  That would
also work for me.

But this might be on my wishlist only...


Bjørn
John Crispin June 5, 2019, 12:33 p.m. UTC | #6
On 05/06/2019 13:35, Karl Palsson wrote:
> John Crispin <john@phrozen.org> wrote:
>> On 05/06/2019 12:17, Karl Palsson wrote:
>>> John Crispin <john@phrozen.org> wrote:
>>>> This can be used inside build setups for easy feeds.conf
>>>> generation.
>>> Could you give us an example of how this is actually easy, or
>>> what sort of functionality this is providing beyond "cat
>>> feeds.conf.default feeds.conf.extra > feeds.conf"
>>>
>>> It seems like a lot of perl for a narrow usecase.
>>>
>>> Sincerely,
>>> Karl Palsson
>> This was brought up as a missing feature by the prpl folks. I
>> considered on how to best implement this and find that having
>> proper tooling is much better than having to carry around an
>> extra file that is cat. being able to build the feeds.conf
>> dynamically like this just seems much cleaner to me and will
>> allow downstream users, vendors, odms and integrators to have
>> less need to patch their trees to death.
> So, they still have to have a script, but now the script has...
>
>
> ...
> ./scripts/feeds setup -b src-git,private-aa,git://blah
> src-link,private-bb,/wop/blah
> ...
>
> instead of
> ...
> cp feeds.conf.default feeds.conf
> echo "src-git private-aa git://blah" >> feeds.conf
> echo "src-link private-bb /wop/blah" >> feeds.conf
> ...
>
> I mean, _yes_ it's "simpler" but it's only simpler by bringing in
> new tools with new layers of abstraction. I really question
> whether that's actually simpler for anyone in the long run, and
> also how this really counts as a "missing feature" There's still
> going to be a requirement for that vendor script.
>
> Sincerely,
> Karl Palsson

Its not a new tool, its an extra call to an already existing one. I 
believe that the one liner is much cleaner than the 3 line scriptage. 
there is no requirement for a vendor script. they ship with a PDF that 
has the build steps. This oneline will be much easier to use I believe.

     John
Jonas Gorski June 5, 2019, 12:54 p.m. UTC | #7
Hi,

On Wed, 5 Jun 2019 at 14:33, John Crispin <john@phrozen.org> wrote:
>
>
> On 05/06/2019 13:35, Karl Palsson wrote:
> > John Crispin <john@phrozen.org> wrote:
> >> On 05/06/2019 12:17, Karl Palsson wrote:
> >>> John Crispin <john@phrozen.org> wrote:
> >>>> This can be used inside build setups for easy feeds.conf
> >>>> generation.
> >>> Could you give us an example of how this is actually easy, or
> >>> what sort of functionality this is providing beyond "cat
> >>> feeds.conf.default feeds.conf.extra > feeds.conf"
> >>>
> >>> It seems like a lot of perl for a narrow usecase.
> >>>
> >>> Sincerely,
> >>> Karl Palsson
> >> This was brought up as a missing feature by the prpl folks. I
> >> considered on how to best implement this and find that having
> >> proper tooling is much better than having to carry around an
> >> extra file that is cat. being able to build the feeds.conf
> >> dynamically like this just seems much cleaner to me and will
> >> allow downstream users, vendors, odms and integrators to have
> >> less need to patch their trees to death.
> > So, they still have to have a script, but now the script has...
> >
> >
> > ...
> > ./scripts/feeds setup -b src-git,private-aa,git://blah
> > src-link,private-bb,/wop/blah
> > ...
> >
> > instead of
> > ...
> > cp feeds.conf.default feeds.conf
> > echo "src-git private-aa git://blah" >> feeds.conf
> > echo "src-link private-bb /wop/blah" >> feeds.conf
> > ...
> >
> > I mean, _yes_ it's "simpler" but it's only simpler by bringing in
> > new tools with new layers of abstraction. I really question
> > whether that's actually simpler for anyone in the long run, and
> > also how this really counts as a "missing feature" There's still
> > going to be a requirement for that vendor script.
> >
> > Sincerely,
> > Karl Palsson
>
> Its not a new tool, its an extra call to an already existing one. I
> believe that the one liner is much cleaner than the 3 line scriptage.
> there is no requirement for a vendor script. they ship with a PDF that
> has the build steps. This oneline will be much easier to use I believe.

Since the use case is having additional custom feeds to the normal
package feeds, maybe it would make more sense to have a e.g.
feeds.conf.custom that is used as an addition to the
feeds.conf.default instead of completely replacing it. That way you
would avoid missing upstream changes in the feeds.conf.default when
updating your build environment.

Then we could add a few commands to scripts/feeds for manipulating
that feeds.conf.custom (adding/removing feeds, changing their
types/addresses/names etc).

We should also sanity check the arguments, as IIRC dashes are actually
not allowed in feed names ;P


Regards
Jonas
John Crispin June 5, 2019, 12:58 p.m. UTC | #8
On 05/06/2019 14:54, Jonas Gorski wrote:
> Hi,
>
> On Wed, 5 Jun 2019 at 14:33, John Crispin <john@phrozen.org> wrote:
>>
>> On 05/06/2019 13:35, Karl Palsson wrote:
>>> John Crispin <john@phrozen.org> wrote:
>>>> On 05/06/2019 12:17, Karl Palsson wrote:
>>>>> John Crispin <john@phrozen.org> wrote:
>>>>>> This can be used inside build setups for easy feeds.conf
>>>>>> generation.
>>>>> Could you give us an example of how this is actually easy, or
>>>>> what sort of functionality this is providing beyond "cat
>>>>> feeds.conf.default feeds.conf.extra > feeds.conf"
>>>>>
>>>>> It seems like a lot of perl for a narrow usecase.
>>>>>
>>>>> Sincerely,
>>>>> Karl Palsson
>>>> This was brought up as a missing feature by the prpl folks. I
>>>> considered on how to best implement this and find that having
>>>> proper tooling is much better than having to carry around an
>>>> extra file that is cat. being able to build the feeds.conf
>>>> dynamically like this just seems much cleaner to me and will
>>>> allow downstream users, vendors, odms and integrators to have
>>>> less need to patch their trees to death.
>>> So, they still have to have a script, but now the script has...
>>>
>>>
>>> ...
>>> ./scripts/feeds setup -b src-git,private-aa,git://blah
>>> src-link,private-bb,/wop/blah
>>> ...
>>>
>>> instead of
>>> ...
>>> cp feeds.conf.default feeds.conf
>>> echo "src-git private-aa git://blah" >> feeds.conf
>>> echo "src-link private-bb /wop/blah" >> feeds.conf
>>> ...
>>>
>>> I mean, _yes_ it's "simpler" but it's only simpler by bringing in
>>> new tools with new layers of abstraction. I really question
>>> whether that's actually simpler for anyone in the long run, and
>>> also how this really counts as a "missing feature" There's still
>>> going to be a requirement for that vendor script.
>>>
>>> Sincerely,
>>> Karl Palsson
>> Its not a new tool, its an extra call to an already existing one. I
>> believe that the one liner is much cleaner than the 3 line scriptage.
>> there is no requirement for a vendor script. they ship with a PDF that
>> has the build steps. This oneline will be much easier to use I believe.
> Since the use case is having additional custom feeds to the normal
> package feeds, maybe it would make more sense to have a e.g.
> feeds.conf.custom that is used as an addition to the
> feeds.conf.default instead of completely replacing it. That way you
> would avoid missing upstream changes in the feeds.conf.default when
> updating your build environment.

Hi,

The patch does not manipulate the default file at all.


>
> Then we could add a few commands to scripts/feeds for manipulating
> that feeds.conf.custom (adding/removing feeds, changing their
> types/addresses/names etc).
so instead of using script/commands to create the already existing 
feeds.conf file we should introduce a 3rd file ? that makes no sense to me.
> We should also sanity check the arguments, as IIRC dashes are actually
> not allowed in feed names ;P

goodd catch, I'll add that to the patch

     John


>
> Regards
> Jonas
>
> _______________________________________________
> openwrt-devel mailing list
> openwrt-devel@lists.openwrt.org
> https://lists.openwrt.org/mailman/listinfo/openwrt-devel
Jonas Gorski June 5, 2019, 1:11 p.m. UTC | #9
On Wed, 5 Jun 2019 at 14:58, John Crispin <john@phrozen.org> wrote:
>
>
> On 05/06/2019 14:54, Jonas Gorski wrote:
> > Hi,
> >
> > On Wed, 5 Jun 2019 at 14:33, John Crispin <john@phrozen.org> wrote:
> >>
> >> On 05/06/2019 13:35, Karl Palsson wrote:
> >>> John Crispin <john@phrozen.org> wrote:
> >>>> On 05/06/2019 12:17, Karl Palsson wrote:
> >>>>> John Crispin <john@phrozen.org> wrote:
> >>>>>> This can be used inside build setups for easy feeds.conf
> >>>>>> generation.
> >>>>> Could you give us an example of how this is actually easy, or
> >>>>> what sort of functionality this is providing beyond "cat
> >>>>> feeds.conf.default feeds.conf.extra > feeds.conf"
> >>>>>
> >>>>> It seems like a lot of perl for a narrow usecase.
> >>>>>
> >>>>> Sincerely,
> >>>>> Karl Palsson
> >>>> This was brought up as a missing feature by the prpl folks. I
> >>>> considered on how to best implement this and find that having
> >>>> proper tooling is much better than having to carry around an
> >>>> extra file that is cat. being able to build the feeds.conf
> >>>> dynamically like this just seems much cleaner to me and will
> >>>> allow downstream users, vendors, odms and integrators to have
> >>>> less need to patch their trees to death.
> >>> So, they still have to have a script, but now the script has...
> >>>
> >>>
> >>> ...
> >>> ./scripts/feeds setup -b src-git,private-aa,git://blah
> >>> src-link,private-bb,/wop/blah
> >>> ...
> >>>
> >>> instead of
> >>> ...
> >>> cp feeds.conf.default feeds.conf
> >>> echo "src-git private-aa git://blah" >> feeds.conf
> >>> echo "src-link private-bb /wop/blah" >> feeds.conf
> >>> ...
> >>>
> >>> I mean, _yes_ it's "simpler" but it's only simpler by bringing in
> >>> new tools with new layers of abstraction. I really question
> >>> whether that's actually simpler for anyone in the long run, and
> >>> also how this really counts as a "missing feature" There's still
> >>> going to be a requirement for that vendor script.
> >>>
> >>> Sincerely,
> >>> Karl Palsson
> >> Its not a new tool, its an extra call to an already existing one. I
> >> believe that the one liner is much cleaner than the 3 line scriptage.
> >> there is no requirement for a vendor script. they ship with a PDF that
> >> has the build steps. This oneline will be much easier to use I believe.
> > Since the use case is having additional custom feeds to the normal
> > package feeds, maybe it would make more sense to have a e.g.
> > feeds.conf.custom that is used as an addition to the
> > feeds.conf.default instead of completely replacing it. That way you
> > would avoid missing upstream changes in the feeds.conf.default when
> > updating your build environment.
>
> Hi,
>
> The patch does not manipulate the default file at all.
>
>
> >
> > Then we could add a few commands to scripts/feeds for manipulating
> > that feeds.conf.custom (adding/removing feeds, changing their
> > types/addresses/names etc).
> so instead of using script/commands to create the already existing
> feeds.conf file we should introduce a 3rd file ? that makes no sense to me.

No, in that case there would be no feeds.conf. Just feeds.conf.default
+ feeds.conf.custom (a "diff"), so still only two files. Different
name to not break existing feeds.conf setups. Or add a marker to
feeds.conf to mark it as a "snippet/diff". Or maybe use the include
thing proposed by Bjørn at the top line of the generated feeds.conf.

So the feeds.conf generated by your command would then be

src-include feeds.conf.default
src-git custom_stuff git://example.com:foo

avoiding having to have a local, unchanging copy of contents of
feeds.conf.default in there.

A bit like we split up the opkg feeds configuration to basic/dist
feeds files and custom feeds file.


Regards
Jonas
John Crispin June 5, 2019, 1:16 p.m. UTC | #10
On 05/06/2019 15:11, Jonas Gorski wrote:
> On Wed, 5 Jun 2019 at 14:58, John Crispin <john@phrozen.org> wrote:
>>
>> On 05/06/2019 14:54, Jonas Gorski wrote:
>>> Hi,
>>>
>>> On Wed, 5 Jun 2019 at 14:33, John Crispin <john@phrozen.org> wrote:
>>>> On 05/06/2019 13:35, Karl Palsson wrote:
>>>>> John Crispin <john@phrozen.org> wrote:
>>>>>> On 05/06/2019 12:17, Karl Palsson wrote:
>>>>>>> John Crispin <john@phrozen.org> wrote:
>>>>>>>> This can be used inside build setups for easy feeds.conf
>>>>>>>> generation.
>>>>>>> Could you give us an example of how this is actually easy, or
>>>>>>> what sort of functionality this is providing beyond "cat
>>>>>>> feeds.conf.default feeds.conf.extra > feeds.conf"
>>>>>>>
>>>>>>> It seems like a lot of perl for a narrow usecase.
>>>>>>>
>>>>>>> Sincerely,
>>>>>>> Karl Palsson
>>>>>> This was brought up as a missing feature by the prpl folks. I
>>>>>> considered on how to best implement this and find that having
>>>>>> proper tooling is much better than having to carry around an
>>>>>> extra file that is cat. being able to build the feeds.conf
>>>>>> dynamically like this just seems much cleaner to me and will
>>>>>> allow downstream users, vendors, odms and integrators to have
>>>>>> less need to patch their trees to death.
>>>>> So, they still have to have a script, but now the script has...
>>>>>
>>>>>
>>>>> ...
>>>>> ./scripts/feeds setup -b src-git,private-aa,git://blah
>>>>> src-link,private-bb,/wop/blah
>>>>> ...
>>>>>
>>>>> instead of
>>>>> ...
>>>>> cp feeds.conf.default feeds.conf
>>>>> echo "src-git private-aa git://blah" >> feeds.conf
>>>>> echo "src-link private-bb /wop/blah" >> feeds.conf
>>>>> ...
>>>>>
>>>>> I mean, _yes_ it's "simpler" but it's only simpler by bringing in
>>>>> new tools with new layers of abstraction. I really question
>>>>> whether that's actually simpler for anyone in the long run, and
>>>>> also how this really counts as a "missing feature" There's still
>>>>> going to be a requirement for that vendor script.
>>>>>
>>>>> Sincerely,
>>>>> Karl Palsson
>>>> Its not a new tool, its an extra call to an already existing one. I
>>>> believe that the one liner is much cleaner than the 3 line scriptage.
>>>> there is no requirement for a vendor script. they ship with a PDF that
>>>> has the build steps. This oneline will be much easier to use I believe.
>>> Since the use case is having additional custom feeds to the normal
>>> package feeds, maybe it would make more sense to have a e.g.
>>> feeds.conf.custom that is used as an addition to the
>>> feeds.conf.default instead of completely replacing it. That way you
>>> would avoid missing upstream changes in the feeds.conf.default when
>>> updating your build environment.
>> Hi,
>>
>> The patch does not manipulate the default file at all.
>>
>>
>>> Then we could add a few commands to scripts/feeds for manipulating
>>> that feeds.conf.custom (adding/removing feeds, changing their
>>> types/addresses/names etc).
>> so instead of using script/commands to create the already existing
>> feeds.conf file we should introduce a 3rd file ? that makes no sense to me.
> No, in that case there would be no feeds.conf. Just feeds.conf.default
> + feeds.conf.custom (a "diff"), so still only two files. Different
> name to not break existing feeds.conf setups. Or add a marker to
> feeds.conf to mark it as a "snippet/diff". Or maybe use the include
> thing proposed by Bjørn at the top line of the generated feeds.conf.
>
> So the feeds.conf generated by your command would then be
>
> src-include feeds.conf.default
> src-git custom_stuff git://example.com:foo
>
> avoiding having to have a local, unchanging copy of contents of
> feeds.conf.default in there.
>
> A bit like we split up the opkg feeds configuration to basic/dist
> feeds files and custom feeds file.
>
>
> Regards
> Jonas


That will yet again require an additional git tree, which is not 
deployable inside a tar file + pdf and is voodoo to the users. I do like 
the idea though, but it is fitting for a foss developer and not a 
corporate coder.

     John
Jonas Gorski June 5, 2019, 1:26 p.m. UTC | #11
On Wed, 5 Jun 2019 at 15:16, John Crispin <john@phrozen.org> wrote:
>
>
> On 05/06/2019 15:11, Jonas Gorski wrote:
> > On Wed, 5 Jun 2019 at 14:58, John Crispin <john@phrozen.org> wrote:
> >>
> >> On 05/06/2019 14:54, Jonas Gorski wrote:
> >>> Hi,
> >>>
> >>> On Wed, 5 Jun 2019 at 14:33, John Crispin <john@phrozen.org> wrote:
> >>>> On 05/06/2019 13:35, Karl Palsson wrote:
> >>>>> John Crispin <john@phrozen.org> wrote:
> >>>>>> On 05/06/2019 12:17, Karl Palsson wrote:
> >>>>>>> John Crispin <john@phrozen.org> wrote:
> >>>>>>>> This can be used inside build setups for easy feeds.conf
> >>>>>>>> generation.
> >>>>>>> Could you give us an example of how this is actually easy, or
> >>>>>>> what sort of functionality this is providing beyond "cat
> >>>>>>> feeds.conf.default feeds.conf.extra > feeds.conf"
> >>>>>>>
> >>>>>>> It seems like a lot of perl for a narrow usecase.
> >>>>>>>
> >>>>>>> Sincerely,
> >>>>>>> Karl Palsson
> >>>>>> This was brought up as a missing feature by the prpl folks. I
> >>>>>> considered on how to best implement this and find that having
> >>>>>> proper tooling is much better than having to carry around an
> >>>>>> extra file that is cat. being able to build the feeds.conf
> >>>>>> dynamically like this just seems much cleaner to me and will
> >>>>>> allow downstream users, vendors, odms and integrators to have
> >>>>>> less need to patch their trees to death.
> >>>>> So, they still have to have a script, but now the script has...
> >>>>>
> >>>>>
> >>>>> ...
> >>>>> ./scripts/feeds setup -b src-git,private-aa,git://blah
> >>>>> src-link,private-bb,/wop/blah
> >>>>> ...
> >>>>>
> >>>>> instead of
> >>>>> ...
> >>>>> cp feeds.conf.default feeds.conf
> >>>>> echo "src-git private-aa git://blah" >> feeds.conf
> >>>>> echo "src-link private-bb /wop/blah" >> feeds.conf
> >>>>> ...
> >>>>>
> >>>>> I mean, _yes_ it's "simpler" but it's only simpler by bringing in
> >>>>> new tools with new layers of abstraction. I really question
> >>>>> whether that's actually simpler for anyone in the long run, and
> >>>>> also how this really counts as a "missing feature" There's still
> >>>>> going to be a requirement for that vendor script.
> >>>>>
> >>>>> Sincerely,
> >>>>> Karl Palsson
> >>>> Its not a new tool, its an extra call to an already existing one. I
> >>>> believe that the one liner is much cleaner than the 3 line scriptage.
> >>>> there is no requirement for a vendor script. they ship with a PDF that
> >>>> has the build steps. This oneline will be much easier to use I believe.
> >>> Since the use case is having additional custom feeds to the normal
> >>> package feeds, maybe it would make more sense to have a e.g.
> >>> feeds.conf.custom that is used as an addition to the
> >>> feeds.conf.default instead of completely replacing it. That way you
> >>> would avoid missing upstream changes in the feeds.conf.default when
> >>> updating your build environment.
> >> Hi,
> >>
> >> The patch does not manipulate the default file at all.
> >>
> >>
> >>> Then we could add a few commands to scripts/feeds for manipulating
> >>> that feeds.conf.custom (adding/removing feeds, changing their
> >>> types/addresses/names etc).
> >> so instead of using script/commands to create the already existing
> >> feeds.conf file we should introduce a 3rd file ? that makes no sense to me.
> > No, in that case there would be no feeds.conf. Just feeds.conf.default
> > + feeds.conf.custom (a "diff"), so still only two files. Different
> > name to not break existing feeds.conf setups. Or add a marker to
> > feeds.conf to mark it as a "snippet/diff". Or maybe use the include
> > thing proposed by Bjørn at the top line of the generated feeds.conf.
> >
> > So the feeds.conf generated by your command would then be
> >
> > src-include feeds.conf.default
> > src-git custom_stuff git://example.com:foo
> >
> > avoiding having to have a local, unchanging copy of contents of
> > feeds.conf.default in there.
> >
> > A bit like we split up the opkg feeds configuration to basic/dist
> > feeds files and custom feeds file.
> >
> >
> > Regards
> > Jonas
>
>
> That will yet again require an additional git tree, which is not
> deployable inside a tar file + pdf and is voodoo to the users. I do like
> the idea though, but it is fitting for a foss developer and not a
> corporate coder.

??? Where does the additional git tree come from?

If the feeds.conf.default doesn't change, that's fine. But not having
the default feeds in a (local) configuration file has the advantage
that if you e.g. update your base distribution/sdk from e.g. 19.06 to
19.12, you don't need to update your feeds.conf to point to the 19.12
branches. Or re-create it.

Jonas
John Crispin June 5, 2019, 1:28 p.m. UTC | #12
On 05/06/2019 15:26, Jonas Gorski wrote:
> On Wed, 5 Jun 2019 at 15:16, John Crispin <john@phrozen.org> wrote:
>>
>> On 05/06/2019 15:11, Jonas Gorski wrote:
>>> On Wed, 5 Jun 2019 at 14:58, John Crispin <john@phrozen.org> wrote:
>>>> On 05/06/2019 14:54, Jonas Gorski wrote:
>>>>> Hi,
>>>>>
>>>>> On Wed, 5 Jun 2019 at 14:33, John Crispin <john@phrozen.org> wrote:
>>>>>> On 05/06/2019 13:35, Karl Palsson wrote:
>>>>>>> John Crispin <john@phrozen.org> wrote:
>>>>>>>> On 05/06/2019 12:17, Karl Palsson wrote:
>>>>>>>>> John Crispin <john@phrozen.org> wrote:
>>>>>>>>>> This can be used inside build setups for easy feeds.conf
>>>>>>>>>> generation.
>>>>>>>>> Could you give us an example of how this is actually easy, or
>>>>>>>>> what sort of functionality this is providing beyond "cat
>>>>>>>>> feeds.conf.default feeds.conf.extra > feeds.conf"
>>>>>>>>>
>>>>>>>>> It seems like a lot of perl for a narrow usecase.
>>>>>>>>>
>>>>>>>>> Sincerely,
>>>>>>>>> Karl Palsson
>>>>>>>> This was brought up as a missing feature by the prpl folks. I
>>>>>>>> considered on how to best implement this and find that having
>>>>>>>> proper tooling is much better than having to carry around an
>>>>>>>> extra file that is cat. being able to build the feeds.conf
>>>>>>>> dynamically like this just seems much cleaner to me and will
>>>>>>>> allow downstream users, vendors, odms and integrators to have
>>>>>>>> less need to patch their trees to death.
>>>>>>> So, they still have to have a script, but now the script has...
>>>>>>>
>>>>>>>
>>>>>>> ...
>>>>>>> ./scripts/feeds setup -b src-git,private-aa,git://blah
>>>>>>> src-link,private-bb,/wop/blah
>>>>>>> ...
>>>>>>>
>>>>>>> instead of
>>>>>>> ...
>>>>>>> cp feeds.conf.default feeds.conf
>>>>>>> echo "src-git private-aa git://blah" >> feeds.conf
>>>>>>> echo "src-link private-bb /wop/blah" >> feeds.conf
>>>>>>> ...
>>>>>>>
>>>>>>> I mean, _yes_ it's "simpler" but it's only simpler by bringing in
>>>>>>> new tools with new layers of abstraction. I really question
>>>>>>> whether that's actually simpler for anyone in the long run, and
>>>>>>> also how this really counts as a "missing feature" There's still
>>>>>>> going to be a requirement for that vendor script.
>>>>>>>
>>>>>>> Sincerely,
>>>>>>> Karl Palsson
>>>>>> Its not a new tool, its an extra call to an already existing one. I
>>>>>> believe that the one liner is much cleaner than the 3 line scriptage.
>>>>>> there is no requirement for a vendor script. they ship with a PDF that
>>>>>> has the build steps. This oneline will be much easier to use I believe.
>>>>> Since the use case is having additional custom feeds to the normal
>>>>> package feeds, maybe it would make more sense to have a e.g.
>>>>> feeds.conf.custom that is used as an addition to the
>>>>> feeds.conf.default instead of completely replacing it. That way you
>>>>> would avoid missing upstream changes in the feeds.conf.default when
>>>>> updating your build environment.
>>>> Hi,
>>>>
>>>> The patch does not manipulate the default file at all.
>>>>
>>>>
>>>>> Then we could add a few commands to scripts/feeds for manipulating
>>>>> that feeds.conf.custom (adding/removing feeds, changing their
>>>>> types/addresses/names etc).
>>>> so instead of using script/commands to create the already existing
>>>> feeds.conf file we should introduce a 3rd file ? that makes no sense to me.
>>> No, in that case there would be no feeds.conf. Just feeds.conf.default
>>> + feeds.conf.custom (a "diff"), so still only two files. Different
>>> name to not break existing feeds.conf setups. Or add a marker to
>>> feeds.conf to mark it as a "snippet/diff". Or maybe use the include
>>> thing proposed by Bjørn at the top line of the generated feeds.conf.
>>>
>>> So the feeds.conf generated by your command would then be
>>>
>>> src-include feeds.conf.default
>>> src-git custom_stuff git://example.com:foo
>>>
>>> avoiding having to have a local, unchanging copy of contents of
>>> feeds.conf.default in there.
>>>
>>> A bit like we split up the opkg feeds configuration to basic/dist
>>> feeds files and custom feeds file.
>>>
>>>
>>> Regards
>>> Jonas
>>
>> That will yet again require an additional git tree, which is not
>> deployable inside a tar file + pdf and is voodoo to the users. I do like
>> the idea though, but it is fitting for a foss developer and not a
>> corporate coder.
> ??? Where does the additional git tree come from?
>
> If the feeds.conf.default doesn't change, that's fine. But not having
> the default feeds in a (local) configuration file has the advantage
> that if you e.g. update your base distribution/sdk from e.g. 19.06 to
> 19.12, you don't need to update your feeds.conf to point to the 19.12
> branches. Or re-create it.
>
> Jonas


ah ok, so i'll modify the patch to not copy the feeds.conf.default to 
feeds.conf but let it reference the file using bjorn's patch

     John
Jonas Gorski June 5, 2019, 1:42 p.m. UTC | #13
On Wed, 5 Jun 2019 at 15:28, John Crispin <john@phrozen.org> wrote:
>
>
> On 05/06/2019 15:26, Jonas Gorski wrote:
> > On Wed, 5 Jun 2019 at 15:16, John Crispin <john@phrozen.org> wrote:
> >>
> >> On 05/06/2019 15:11, Jonas Gorski wrote:
> >>> On Wed, 5 Jun 2019 at 14:58, John Crispin <john@phrozen.org> wrote:
> >>>> On 05/06/2019 14:54, Jonas Gorski wrote:
> >>>>> Hi,
> >>>>>
> >>>>> On Wed, 5 Jun 2019 at 14:33, John Crispin <john@phrozen.org> wrote:
> >>>>>> On 05/06/2019 13:35, Karl Palsson wrote:
> >>>>>>> John Crispin <john@phrozen.org> wrote:
> >>>>>>>> On 05/06/2019 12:17, Karl Palsson wrote:
> >>>>>>>>> John Crispin <john@phrozen.org> wrote:
> >>>>>>>>>> This can be used inside build setups for easy feeds.conf
> >>>>>>>>>> generation.
> >>>>>>>>> Could you give us an example of how this is actually easy, or
> >>>>>>>>> what sort of functionality this is providing beyond "cat
> >>>>>>>>> feeds.conf.default feeds.conf.extra > feeds.conf"
> >>>>>>>>>
> >>>>>>>>> It seems like a lot of perl for a narrow usecase.
> >>>>>>>>>
> >>>>>>>>> Sincerely,
> >>>>>>>>> Karl Palsson
> >>>>>>>> This was brought up as a missing feature by the prpl folks. I
> >>>>>>>> considered on how to best implement this and find that having
> >>>>>>>> proper tooling is much better than having to carry around an
> >>>>>>>> extra file that is cat. being able to build the feeds.conf
> >>>>>>>> dynamically like this just seems much cleaner to me and will
> >>>>>>>> allow downstream users, vendors, odms and integrators to have
> >>>>>>>> less need to patch their trees to death.
> >>>>>>> So, they still have to have a script, but now the script has...
> >>>>>>>
> >>>>>>>
> >>>>>>> ...
> >>>>>>> ./scripts/feeds setup -b src-git,private-aa,git://blah
> >>>>>>> src-link,private-bb,/wop/blah
> >>>>>>> ...
> >>>>>>>
> >>>>>>> instead of
> >>>>>>> ...
> >>>>>>> cp feeds.conf.default feeds.conf
> >>>>>>> echo "src-git private-aa git://blah" >> feeds.conf
> >>>>>>> echo "src-link private-bb /wop/blah" >> feeds.conf
> >>>>>>> ...
> >>>>>>>
> >>>>>>> I mean, _yes_ it's "simpler" but it's only simpler by bringing in
> >>>>>>> new tools with new layers of abstraction. I really question
> >>>>>>> whether that's actually simpler for anyone in the long run, and
> >>>>>>> also how this really counts as a "missing feature" There's still
> >>>>>>> going to be a requirement for that vendor script.
> >>>>>>>
> >>>>>>> Sincerely,
> >>>>>>> Karl Palsson
> >>>>>> Its not a new tool, its an extra call to an already existing one. I
> >>>>>> believe that the one liner is much cleaner than the 3 line scriptage.
> >>>>>> there is no requirement for a vendor script. they ship with a PDF that
> >>>>>> has the build steps. This oneline will be much easier to use I believe.
> >>>>> Since the use case is having additional custom feeds to the normal
> >>>>> package feeds, maybe it would make more sense to have a e.g.
> >>>>> feeds.conf.custom that is used as an addition to the
> >>>>> feeds.conf.default instead of completely replacing it. That way you
> >>>>> would avoid missing upstream changes in the feeds.conf.default when
> >>>>> updating your build environment.
> >>>> Hi,
> >>>>
> >>>> The patch does not manipulate the default file at all.
> >>>>
> >>>>
> >>>>> Then we could add a few commands to scripts/feeds for manipulating
> >>>>> that feeds.conf.custom (adding/removing feeds, changing their
> >>>>> types/addresses/names etc).
> >>>> so instead of using script/commands to create the already existing
> >>>> feeds.conf file we should introduce a 3rd file ? that makes no sense to me.
> >>> No, in that case there would be no feeds.conf. Just feeds.conf.default
> >>> + feeds.conf.custom (a "diff"), so still only two files. Different
> >>> name to not break existing feeds.conf setups. Or add a marker to
> >>> feeds.conf to mark it as a "snippet/diff". Or maybe use the include
> >>> thing proposed by Bjørn at the top line of the generated feeds.conf.
> >>>
> >>> So the feeds.conf generated by your command would then be
> >>>
> >>> src-include feeds.conf.default
> >>> src-git custom_stuff git://example.com:foo
> >>>
> >>> avoiding having to have a local, unchanging copy of contents of
> >>> feeds.conf.default in there.
> >>>
> >>> A bit like we split up the opkg feeds configuration to basic/dist
> >>> feeds files and custom feeds file.
> >>>
> >>>
> >>> Regards
> >>> Jonas
> >>
> >> That will yet again require an additional git tree, which is not
> >> deployable inside a tar file + pdf and is voodoo to the users. I do like
> >> the idea though, but it is fitting for a foss developer and not a
> >> corporate coder.
> > ??? Where does the additional git tree come from?
> >
> > If the feeds.conf.default doesn't change, that's fine. But not having
> > the default feeds in a (local) configuration file has the advantage
> > that if you e.g. update your base distribution/sdk from e.g. 19.06 to
> > 19.12, you don't need to update your feeds.conf to point to the 19.12
> > branches. Or re-create it.
> >
> > Jonas
>
>
> ah ok, so i'll modify the patch to not copy the feeds.conf.default to
> feeds.conf but let it reference the file using bjorn's patch

Exactly what I was proposing :)

Eventually we might want to not require to pass all custom feeds at
once to the script, but have a add_feed command (so you can easily add
one at a later time), but that is nothing that affects your proposed
changes, just adds on top of it.


Jonas

Patch
diff mbox series

diff --git a/scripts/feeds b/scripts/feeds
index 304ef6cbaf..7cd4639ca6 100755
--- a/scripts/feeds
+++ b/scripts/feeds
@@ -7,6 +7,7 @@  use metadata;
 use warnings;
 use strict;
 use Cwd 'abs_path';
+use File::Copy;
 
 chdir "$FindBin::Bin/..";
 $ENV{TOPDIR} //= getcwd();
@@ -819,6 +820,42 @@  sub update {
 	return $failed;
 }
 
+sub setup {
+	my %opts;
+
+	getopts('bh', \%opts);
+
+	if ($opts{h}) {
+		usage();
+		return 0;
+	}
+
+	if ($opts{b}) {
+		copy("feeds.conf.default", "feeds.conf") or die "Copy failed: $!"
+	} else {
+		unlink "feeds.conf"
+	}
+
+	open(my $fd, ">>feeds.conf");
+	while (my $entry = shift @ARGV) {
+		my ($type, $name, $src) = split /,/, $entry;
+
+		$update_method{$type} or do {
+			warn "Unknown type '$type' in parameter $entry\n";
+			unlink "feeds.conf";
+			return 1;
+		};
+		if ($name =~ /\s/ || $src =~ /\s/) {
+			warn "Feed names or sources may not contain whitespace characters in parameter $entry\n";
+			unlink "feeds.conf";
+			return 1;
+		}
+		printf $fd "%s %s %s\n", $type, $name, $src;
+	}
+
+	return 0;
+}
+
 sub feed_config() {
 	foreach my $feed (@feeds) {
 		my $installed = (-f "feeds/$feed->[1].index");
@@ -870,6 +907,10 @@  Commands:
 	    -i :           Recreate the index only. No feed update from repository is performed.
 	    -f :           Force updating feeds even if there are changed, uncommitted files.
 
+	setup [options] <type,name,link> <type,name,link> ...: generate feeds.conf
+	Options:
+	    -b :           Use feeds.conf.default as base for new feeds.conf.
+
 	clean:             Remove downloaded/generated files.
 
 EOF
@@ -883,6 +924,7 @@  my %commands = (
 	'search' => \&search,
 	'uninstall' => \&uninstall,
 	'feed_config' => \&feed_config,
+	'setup' => \&setup,
 	'clean' => sub {
 		system("rm -rf ./feeds ./package/feeds");
 	}