[1/3] Add export project as mbox management command
diff mbox series

Message ID 20190715145030.7877-1-metepolat2000@gmail.com
State New
Headers show
Series
  • [1/3] Add export project as mbox management command
Related show

Commit Message

Mete Polat July 15, 2019, 2:50 p.m. UTC
Introduces a new management command which can export all patches in a
project as one mbox file. Export of multiple projects is supported.
Additionaly allows to compress the output.

Signed-off-by: Mete Polat <metepolat2000@gmail.com>
---
Also supports python2 with the force_bytes() function.

 .../management/commands/exportproject.py      | 71 +++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 patchwork/management/commands/exportproject.py

Comments

Stephen Finucane July 15, 2019, 3:39 p.m. UTC | #1
On Mon, 2019-07-15 at 16:50 +0200, Mete Polat wrote:
> Introduces a new management command which can export all patches in a
> project as one mbox file. Export of multiple projects is supported.
> Additionaly allows to compress the output.

This looks good. Just two small comments below.

> Signed-off-by: Mete Polat <metepolat2000@gmail.com>
> ---
> Also supports python2 with the force_bytes() function.
> 
>  .../management/commands/exportproject.py      | 71 +++++++++++++++++++
>  1 file changed, 71 insertions(+)
>  create mode 100644 patchwork/management/commands/exportproject.py
> 
> diff --git a/patchwork/management/commands/exportproject.py b/patchwork/management/commands/exportproject.py
> new file mode 100644
> index 0000000..7e18234
> --- /dev/null
> +++ b/patchwork/management/commands/exportproject.py
> @@ -0,0 +1,71 @@
> +# Patchwork - automated patch tracking system
> +# Copyright (C) 2019, Bayerische Motoren Werke Aktiengesellschaft (BMW AG) 
> +#
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +
> +import os
> +import tarfile
> +from uuid import uuid4
> +
> +from django.core.management import BaseCommand, CommandError
> +from django.utils.encoding import force_bytes
> +
> +from patchwork.models import Project, Patch
> +from patchwork.views.utils import patch_to_mbox
> +
> +
> +class Command(BaseCommand):
> +    help = 'Export patchwork projects as mbox files and optionally compress '
> +    'the result.'
> +
> +    def add_arguments(self, parser):
> +        parser.add_argument(
> +            '-c', '--compress', action='store_true',
> +            help='Bundle and compress projects.'
> +        )
> +        parser.add_argument(
> +            '-l', '--level', action='store', type=int, default=9,
> +            help='Set a compression level between 0 and 9 (default). 0 is no '
> +                 'compression. '
> +        )

Do we need this knob? It seems defaulting to 9 if the '-c' flag is
provided would be fine for 90% of users?

> +        parser.add_argument(
> +            'project_linkname', nargs='*',
> +            help='Project linknames. Export all projects if none specified.'
> +        )
> +
> +    def handle(self, *args, **options):
> +        if options['project_linkname']:
> +            projects = []
> +            for p_linkname in options['project_linkname']:
> +                try:
> +                    projects.append(Project.objects.get(linkname=p_linkname))
> +                except Project.DoesNotExist:
> +                    raise CommandError('%s: Project not found' % p_linkname)
> +        else:
> +            projects = Project.objects.all()
> +
> +        compress = options['compress']
> +        level = options['level']
> +
> +        tar = None
> +        if compress:
> +            name = projects[0].linkname if len(projects) == 1 else 'patchwork'

Any reason not to use 'projects[0].linkname' if len(projects) == 1 too?

> +            name += '.tar' if level == 0 else '.tar.gz'
> +            tar = tarfile.open(name, 'w:gz', compresslevel=level)
> +
> +        try:
> +            for project in projects:
> +                name = project.linkname + '.mbox'
> +                tmp_name = '%s_%s' % (project.linkname, uuid4().hex)
> +                with open(tmp_name, 'wb') as mbox:
> +                    for patch in Patch.objects.filter(patch_project=project):
> +                        mbox.write(force_bytes(patch_to_mbox(patch) + '\n'))
> +                    mbox.close()
> +                    if compress:
> +                        tar.add(tmp_name, arcname=name)
> +                        os.remove(tmp_name)
> +                    else:
> +                        os.rename(tmp_name, name)
> +        finally:
> +            if tar is not None:
> +                tar.close()
Mete Polat July 16, 2019, 11:27 a.m. UTC | #2
On 15.07.19 17:39, Stephen Finucane wrote:
> On Mon, 2019-07-15 at 16:50 +0200, Mete Polat wrote:
>> Introduces a new management command which can export all patches in a
>> project as one mbox file. Export of multiple projects is supported.
>> Additionaly allows to compress the output.
> This looks good. Just two small comments below.
>
>> Signed-off-by: Mete Polat <metepolat2000@gmail.com>
>> ---
>> Also supports python2 with the force_bytes() function.
>>
>>   .../management/commands/exportproject.py      | 71 +++++++++++++++++++
>>   1 file changed, 71 insertions(+)
>>   create mode 100644 patchwork/management/commands/exportproject.py
>>
>> diff --git a/patchwork/management/commands/exportproject.py b/patchwork/management/commands/exportproject.py
>> new file mode 100644
>> index 0000000..7e18234
>> --- /dev/null
>> +++ b/patchwork/management/commands/exportproject.py
>> @@ -0,0 +1,71 @@
>> +# Patchwork - automated patch tracking system
>> +# Copyright (C) 2019, Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
>> +#
>> +# SPDX-License-Identifier: GPL-2.0-or-later
>> +
>> +import os
>> +import tarfile
>> +from uuid import uuid4
>> +
>> +from django.core.management import BaseCommand, CommandError
>> +from django.utils.encoding import force_bytes
>> +
>> +from patchwork.models import Project, Patch
>> +from patchwork.views.utils import patch_to_mbox
>> +
>> +
>> +class Command(BaseCommand):
>> +    help = 'Export patchwork projects as mbox files and optionally compress '
>> +    'the result.'
>> +
>> +    def add_arguments(self, parser):
>> +        parser.add_argument(
>> +            '-c', '--compress', action='store_true',
>> +            help='Bundle and compress projects.'
>> +        )
>> +        parser.add_argument(
>> +            '-l', '--level', action='store', type=int, default=9,
>> +            help='Set a compression level between 0 and 9 (default). 0 is no '
>> +                 'compression. '
>> +        )
> Do we need this knob? It seems defaulting to 9 if the '-c' flag is
> provided would be fine for 90% of users?

Indeed, fine-tuning the compression level will be hardly used but 
especially for
large projects turning off compression with '-l 0' can save time when 
there is
no need for sharing the output.

>> +        parser.add_argument(
>> +            'project_linkname', nargs='*',
>> +            help='Project linknames. Export all projects if none specified.'
>> +        )
>> +
>> +    def handle(self, *args, **options):
>> +        if options['project_linkname']:
>> +            projects = []
>> +            for p_linkname in options['project_linkname']:
>> +                try:
>> +                    projects.append(Project.objects.get(linkname=p_linkname))
>> +                except Project.DoesNotExist:
>> +                    raise CommandError('%s: Project not found' % p_linkname)
>> +        else:
>> +            projects = Project.objects.all()
>> +
>> +        compress = options['compress']
>> +        level = options['level']
>> +
>> +        tar = None
>> +        if compress:
>> +            name = projects[0].linkname if len(projects) == 1 else 'patchwork'
> Any reason not to use 'projects[0].linkname' if len(projects) == 1 too?

Do you mean 'len(projects) != 1'? It can be confusing if we use the name of
the first project as the archive name when there is more than one 
project in it.

>> +            name += '.tar' if level == 0 else '.tar.gz'
>> +            tar = tarfile.open(name, 'w:gz', compresslevel=level)
>> +
>> +        try:
>> +            for project in projects:
>> +                name = project.linkname + '.mbox'
>> +                tmp_name = '%s_%s' % (project.linkname, uuid4().hex)
>> +                with open(tmp_name, 'wb') as mbox:
>> +                    for patch in Patch.objects.filter(patch_project=project):
>> +                        mbox.write(force_bytes(patch_to_mbox(patch) + '\n'))
>> +                    mbox.close()
>> +                    if compress:
>> +                        tar.add(tmp_name, arcname=name)
>> +                        os.remove(tmp_name)
>> +                    else:
>> +                        os.rename(tmp_name, name)
>> +        finally:
>> +            if tar is not None:
>> +                tar.close()
Stephen Finucane July 19, 2019, 1:50 p.m. UTC | #3
[re-adding the list]

On Fri, 2019-07-19 at 14:44 +0200, Lukas Bulwahn wrote:
> On Mon, Jul 15, 2019 at 5:39 PM Stephen Finucane <stephen@that.guru> wrote:
> > On Mon, 2019-07-15 at 16:50 +0200, Mete Polat wrote:

[snip]

> > > +    def add_arguments(self, parser):
> > > +        parser.add_argument(
> > > +            '-c', '--compress', action='store_true',
> > > +            help='Bundle and compress projects.'
> > > +        )
> > > +        parser.add_argument(
> > > +            '-l', '--level', action='store', type=int, default=9,
> > > +            help='Set a compression level between 0 and 9 (default). 0 is no '
> > > +                 'compression. '
> > > +        )
> > 
> > Do we need this knob? It seems defaulting to 9 if the '-c' flag is
> > provided would be fine for 90% of users?
> 
> I am wondering if we need compression at all.
> 
> Following the UNIX philosophy (only do Make each program do one thing
> well.), I would expect that the management command can either generate
> files/an archive file or I can configure it to output the file/the
> archive to standard output.
> I can then simply compress by pipe-ing that to any other compression utility.
> 
> Stephen, Mete, what do you think?

That's a fair point but I'm not sure how we could support both dumping
to stdout and multiple projects: how would you decide which patches
belonged to which project? Looking at the logic here though, I do think
we could tweak it slightly. Could we *always* generate a tarball,
regardless of the number of projects, and optionally compress it if we
decide to keep the '--compress' flag? I've drafted this and it looks
good to me. I also think we might want to rename the command
'dumparchive' to match with 'parsearchive'. I'm happy to rework on both
counts. Thoughts?

Stephen
Mete Polat July 19, 2019, 2:37 p.m. UTC | #4
On 19.07.19 15:50, Stephen Finucane wrote:
> [re-adding the list]
>
> On Fri, 2019-07-19 at 14:44 +0200, Lukas Bulwahn wrote:
>> On Mon, Jul 15, 2019 at 5:39 PM Stephen Finucane <stephen@that.guru> wrote:
>>> On Mon, 2019-07-15 at 16:50 +0200, Mete Polat wrote:
> [snip]
>
>>>> +    def add_arguments(self, parser):
>>>> +        parser.add_argument(
>>>> +            '-c', '--compress', action='store_true',
>>>> +            help='Bundle and compress projects.'
>>>> +        )
>>>> +        parser.add_argument(
>>>> +            '-l', '--level', action='store', type=int, default=9,
>>>> +            help='Set a compression level between 0 and 9 (default). 0 is no '
>>>> +                 'compression. '
>>>> +        )
>>> Do we need this knob? It seems defaulting to 9 if the '-c' flag is
>>> provided would be fine for 90% of users?
>> I am wondering if we need compression at all.
>>
>> Following the UNIX philosophy (only do Make each program do one thing
>> well.), I would expect that the management command can either generate
>> files/an archive file or I can configure it to output the file/the
>> archive to standard output.
>> I can then simply compress by pipe-ing that to any other compression utility.
>>
>> Stephen, Mete, what do you think?
> That's a fair point but I'm not sure how we could support both dumping
> to stdout and multiple projects: how would you decide which patches
> belonged to which project? Looking at the logic here though, I do think
> we could tweak it slightly. Could we *always* generate a tarball,
> regardless of the number of projects, and optionally compress it if we
> decide to keep the '--compress' flag? I've drafted this and it looks
> good to me. I also think we might want to rename the command
> 'dumparchive' to match with 'parsearchive'. I'm happy to rework on both
> counts. Thoughts?
>
> Stephen
>

I agree you Stephen on both points. We should also consider using the 
project's
listId instead of the linkname in order to match with the other 
commands. Feel
free to implement your changes when Lukas also agrees on your points.

Mete
Lukas Bulwahn July 19, 2019, 4:21 p.m. UTC | #5
On Fri, Jul 19, 2019 at 3:50 PM Stephen Finucane <stephen@that.guru> wrote:
>
> [re-adding the list]

Oops... my mistake; my email client confused me.

>
> On Fri, 2019-07-19 at 14:44 +0200, Lukas Bulwahn wrote:
> > On Mon, Jul 15, 2019 at 5:39 PM Stephen Finucane <stephen@that.guru> wrote:
> > > On Mon, 2019-07-15 at 16:50 +0200, Mete Polat wrote:
>
> [snip]
>
> > > > +    def add_arguments(self, parser):
> > > > +        parser.add_argument(
> > > > +            '-c', '--compress', action='store_true',
> > > > +            help='Bundle and compress projects.'
> > > > +        )
> > > > +        parser.add_argument(
> > > > +            '-l', '--level', action='store', type=int, default=9,
> > > > +            help='Set a compression level between 0 and 9 (default). 0 is no '
> > > > +                 'compression. '
> > > > +        )
> > >
> > > Do we need this knob? It seems defaulting to 9 if the '-c' flag is
> > > provided would be fine for 90% of users?
> >
> > I am wondering if we need compression at all.
> >
> > Following the UNIX philosophy (only do Make each program do one thing
> > well.), I would expect that the management command can either generate
> > files/an archive file or I can configure it to output the file/the
> > archive to standard output.
> > I can then simply compress by pipe-ing that to any other compression utility.
> >
> > Stephen, Mete, what do you think?
>
> That's a fair point but I'm not sure how we could support both dumping
> to stdout and multiple projects: how would you decide which patches
> belonged to which project? Looking at the logic here though, I do think
> we could tweak it slightly. Could we *always* generate a tarball,
> regardless of the number of projects, and optionally compress it if we
> decide to keep the '--compress' flag? I've drafted this and it looks
> good to me. I also think we might want to rename the command
> 'dumparchive' to match with 'parsearchive'. I'm happy to rework on both
> counts. Thoughts?
>
> Stephen
>
Always creating a tarball sounds good.
I am still in favor of having it print to stdout and simply pipe it in
the compression tool of your choice.

But Stephen go ahead and implement as you think that is best.

Lukas

Patch
diff mbox series

diff --git a/patchwork/management/commands/exportproject.py b/patchwork/management/commands/exportproject.py
new file mode 100644
index 0000000..7e18234
--- /dev/null
+++ b/patchwork/management/commands/exportproject.py
@@ -0,0 +1,71 @@ 
+# Patchwork - automated patch tracking system
+# Copyright (C) 2019, Bayerische Motoren Werke Aktiengesellschaft (BMW AG) 
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import os
+import tarfile
+from uuid import uuid4
+
+from django.core.management import BaseCommand, CommandError
+from django.utils.encoding import force_bytes
+
+from patchwork.models import Project, Patch
+from patchwork.views.utils import patch_to_mbox
+
+
+class Command(BaseCommand):
+    help = 'Export patchwork projects as mbox files and optionally compress '
+    'the result.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-c', '--compress', action='store_true',
+            help='Bundle and compress projects.'
+        )
+        parser.add_argument(
+            '-l', '--level', action='store', type=int, default=9,
+            help='Set a compression level between 0 and 9 (default). 0 is no '
+                 'compression. '
+        )
+        parser.add_argument(
+            'project_linkname', nargs='*',
+            help='Project linknames. Export all projects if none specified.'
+        )
+
+    def handle(self, *args, **options):
+        if options['project_linkname']:
+            projects = []
+            for p_linkname in options['project_linkname']:
+                try:
+                    projects.append(Project.objects.get(linkname=p_linkname))
+                except Project.DoesNotExist:
+                    raise CommandError('%s: Project not found' % p_linkname)
+        else:
+            projects = Project.objects.all()
+
+        compress = options['compress']
+        level = options['level']
+
+        tar = None
+        if compress:
+            name = projects[0].linkname if len(projects) == 1 else 'patchwork'
+            name += '.tar' if level == 0 else '.tar.gz'
+            tar = tarfile.open(name, 'w:gz', compresslevel=level)
+
+        try:
+            for project in projects:
+                name = project.linkname + '.mbox'
+                tmp_name = '%s_%s' % (project.linkname, uuid4().hex)
+                with open(tmp_name, 'wb') as mbox:
+                    for patch in Patch.objects.filter(patch_project=project):
+                        mbox.write(force_bytes(patch_to_mbox(patch) + '\n'))
+                    mbox.close()
+                    if compress:
+                        tar.add(tmp_name, arcname=name)
+                        os.remove(tmp_name)
+                    else:
+                        os.rename(tmp_name, name)
+        finally:
+            if tar is not None:
+                tar.close()