summaryrefslogtreecommitdiff
path: root/support/scripts/scanpypi
blob: e150ae588d2baeced47bc5b3c85299772fcf36b5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
#!/usr/bin/python2
"""

Utility for building Buildroot packages for existing PyPI packages

Any package built by scanpypi should be manually checked for
errors.
"""
from __future__ import print_function
import argparse
import json
import urllib2
import sys
import os
import shutil
import StringIO
import tarfile
import zipfile
import errno
import hashlib
import re
import textwrap
import tempfile
import imp
from functools import wraps

def setup_decorator(func, method):
    """
    Decorator for distutils.core.setup and setuptools.setup.
    Puts the arguments with which setup is called as a dict
    Add key 'method' which should be either 'setuptools' or 'distutils'.

    Keyword arguments:
    func -- either setuptools.setup or distutils.core.setup
    method -- either 'setuptools' or 'distutils'
    """

    @wraps(func)
    def closure(*args, **kwargs):
        # Any python packages calls its setup function to be installed.
        # Argument 'name' of this setup function is the package's name
        BuildrootPackage.setup_args[kwargs['name']] = kwargs
        BuildrootPackage.setup_args[kwargs['name']]['method'] = method
    return closure

# monkey patch
import setuptools
setuptools.setup = setup_decorator(setuptools.setup, 'setuptools')
import distutils
distutils.core.setup = setup_decorator(setuptools.setup, 'distutils')

def find_file_upper_case(filenames, path='./'):
    """
    List generator:
    Recursively find files that matches one of the specified filenames.
    Returns a relative path starting with path argument.

    Keyword arguments:
    filenames -- List of filenames to be found
    path -- Path to the directory to search
    """
    for root, dirs, files in os.walk(path):
        for file in files:
            if file.upper() in filenames:
                yield (os.path.join(root, file))


def pkg_buildroot_name(pkg_name):
    """
    Returns the Buildroot package name for the PyPI package pkg_name.
    Remove all non alphanumeric characters except -
    Also lowers the name and adds 'python-' suffix

    Keyword arguments:
    pkg_name -- String to rename
    """
    name = re.sub('[^\w-]', '', pkg_name.lower())
    prefix = 'python-'
    pattern = re.compile('^(?!' + prefix + ')(.+?)$')
    name = pattern.sub(r'python-\1', name)
    return name

class DownloadFailed(Exception):
    pass

class BuildrootPackage():
    """This class's methods are not meant to be used individually please
    use them in the correct order:

    __init__

    download_package

    extract_package

    load_module

    get_requirements

    create_package_mk

    create_hash_file

    create_config_in

    """
    setup_args = {}

    def __init__(self, real_name, pkg_folder):
        self.real_name = real_name
        self.buildroot_name = pkg_buildroot_name(self.real_name)
        self.pkg_dir = os.path.join(pkg_folder, self.buildroot_name)
        self.mk_name = self.buildroot_name.upper().replace('-', '_')
        self.as_string = None
        self.md5_sum = None
        self.metadata = None
        self.metadata_name = None
        self.metadata_url = None
        self.pkg_req = None
        self.setup_metadata = None
        self.tmp_extract = None
        self.used_url = None
        self.filename = None
        self.url = None
        self.version = None

    def fetch_package_info(self):
        """
        Fetch a package's metadata from the python package index
        """
        self.metadata_url = 'https://pypi.python.org/pypi/{pkg}/json'.format(
            pkg=self.real_name)
        try:
            pkg_json = urllib2.urlopen(self.metadata_url).read().decode()
        except urllib2.HTTPError as error:
            print('ERROR:', error.getcode(), error.msg, file=sys.stderr)
            print('ERROR: Could not find package {pkg}.\n'
                  'Check syntax inside the python package index:\n'
                  'https://pypi.python.org/pypi/ '
                  .format(pkg=self.real_name))
            raise
        except urllib2.URLError:
            print('ERROR: Could not find package {pkg}.\n'
                  'Check syntax inside the python package index:\n'
                  'https://pypi.python.org/pypi/ '
                  .format(pkg=self.real_name))
            raise
        self.metadata = json.loads(pkg_json)
        self.version = self.metadata['info']['version']
        self.metadata_name = self.metadata['info']['name']

    def download_package(self):
        """
        Download a package using metadata from pypi
        """
        try:
            self.metadata['urls'][0]['filename']
        except IndexError:
            print(
                'Non-conventional package, ',
                'please check carefully after creation')
            self.metadata['urls'] = [{
                'packagetype': 'sdist',
                'url': self.metadata['info']['download_url'],
                'md5_digest': None}]
            # In this case, we can't get the name of the downloaded file
            # from the pypi api, so we need to find it, this should work
            urlpath = urllib2.urlparse.urlparse(
                self.metadata['info']['download_url']).path
            # urlparse().path give something like
            # /path/to/file-version.tar.gz
            # We use basename to remove /path/to
            self.metadata['urls'][0]['filename'] = os.path.basename(urlpath)
        for download_url in self.metadata['urls']:
            if 'bdist' in download_url['packagetype']:
                continue
            try:
                print('Downloading package {pkg} from {url}...'.format(
                      pkg=self.real_name, url=download_url['url']))
                download = urllib2.urlopen(download_url['url'])
            except urllib2.HTTPError as http_error:
                download = http_error
            else:
                self.used_url = download_url
                self.as_string = download.read()
                if not download_url['md5_digest']:
                    break
                self.md5_sum = hashlib.md5(self.as_string).hexdigest()
                if self.md5_sum == download_url['md5_digest']:
                    break
        else:
            if download.__class__ == urllib2.HTTPError:
                raise download
            raise DownloadFailed('Failed to downloas package {pkg}'
                                 .format(pkg=self.real_name))
        self.filename = self.used_url['filename']
        self.url = self.used_url['url']

    def extract_package(self, tmp_path):
        """
        Extract the package contents into a directrory

        Keyword arguments:
        tmp_path -- directory where you want the package to be extracted
        """
        as_file = StringIO.StringIO(self.as_string)
        if self.filename[-3:] == 'zip':
            with zipfile.ZipFile(as_file) as as_zipfile:
                tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
                try:
                    os.makedirs(tmp_pkg)
                except OSError as exception:
                    if exception.errno != errno.EEXIST:
                        print("ERROR: ", exception.message, file=sys.stderr)
                        return None, None
                    print('WARNING:', exception.message, file=sys.stderr)
                    print('Removing {pkg}...'.format(pkg=tmp_pkg))
                    shutil.rmtree(tmp_pkg)
                    os.makedirs(tmp_pkg)
                as_zipfile.extractall(tmp_pkg)
        else:
            with tarfile.open(fileobj=as_file) as as_tarfile:
                tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
                try:
                    os.makedirs(tmp_pkg)
                except OSError as exception:
                    if exception.errno != errno.EEXIST:
                        print("ERROR: ", exception.message, file=sys.stderr)
                        return None, None
                    print('WARNING:', exception.message, file=sys.stderr)
                    print('Removing {pkg}...'.format(pkg=tmp_pkg))
                    shutil.rmtree(tmp_pkg)
                    os.makedirs(tmp_pkg)
                as_tarfile.extractall(tmp_pkg)

        tmp_extract = '{folder}/{name}-{version}'
        self.tmp_extract = tmp_extract.format(
            folder=tmp_pkg,
            name=self.metadata_name,
            version=self.version)

    def load_setup(self):
        """
        Loads the corresponding setup and store its metadata
        """
        current_dir = os.getcwd()
        os.chdir(self.tmp_extract)
        sys.path.append(self.tmp_extract)
        s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
        setup = imp.load_module('setup', s_file, s_path, s_desc)
        try:
            self.setup_metadata = self.setup_args[self.metadata_name]
        except KeyError:
            # This means setup was not called which most likely mean that it is
            # called through the if __name__ == '__main__' directive.
            # In this case, we can only pray that it is called through a
            # function called main() in setup.py.
            setup.main([]) # Will raise AttributeError if not found
            self.setup_metadata = self.setup_args[self.metadata_name]
        # Here we must remove the module the hard way.
        # We must do this because of a very sepcific case: if a package calls
        # setup from the __main__ but does not come with a 'main()' function,
        # for some reason setup.main([]) will successfully call the main
        # function of a previous package...
        sys.modules.pop('setup',None)
        del setup
        os.chdir(current_dir)
        sys.path.remove(self.tmp_extract)

    def get_requirements(self, pkg_folder):
        """
        Retrieve dependencies from the metadata found in the setup.py script of
        a pypi package.

        Keyword Arguments:
        pkg_folder -- location of the already created packages
        """
        if 'install_requires' not in self.setup_metadata:
            self.pkg_req = None
            return set()
        self.pkg_req = self.setup_metadata['install_requires']
        self.pkg_req = [re.sub('([-.\w]+).*', r'\1', req)
                        for req in self.pkg_req]
        req_not_found = self.pkg_req
        self.pkg_req = map(pkg_buildroot_name, self.pkg_req)
        pkg_tuples = zip(req_not_found, self.pkg_req)
        # pkg_tuples is a list of tuples that looks like
        # ('werkzeug','python-werkzeug') because I need both when checking if
        # dependencies already exist or are already in the download list
        req_not_found = set(
            pkg[0] for pkg in pkg_tuples
            if not os.path.isdir(pkg[1])
            )
        return req_not_found

    def __create_mk_header(self):
        """
        Create the header of the <package_name>.mk file
        """
        header = ['#' * 80 + '\n']
        header.append('#\n')
        header.append('# {name}\n'.format(name=self.buildroot_name))
        header.append('#\n')
        header.append('#' * 80 + '\n')
        header.append('\n')
        return header

    def __create_mk_download_info(self):
        """
        Create the lines refering to the download information of the
        <package_name>.mk file
        """
        lines = []
        version_line = '{name}_VERSION = {version}\n'.format(
            name=self.mk_name,
            version=self.version)
        lines.append(version_line)

        targz = self.filename.replace(
            self.version,
            '$({name}_VERSION)'.format(name=self.mk_name))
        targz_line = '{name}_SOURCE = {filename}\n'.format(
            name=self.mk_name,
            filename=targz)
        lines.append(targz_line)

        if self.filename not in self.url:
            # Sometimes the filename is in the url, sometimes it's not
            site_url = self.url
        else:
            site_url = self.url[:self.url.find(self.filename)]
        site_line = '{name}_SITE = {url}'.format(name=self.mk_name,
                                                 url=site_url)
        site_line = site_line.rstrip('/') + '\n'
        lines.append(site_line)
        return lines

    def __create_mk_setup(self):
        """
        Create the line refering to the setup method of the package of the
        <package_name>.mk file

        There are two things you can use to make an installer
        for a python package: distutils or setuptools
        distutils comes with python but does not support dependencies.
        distutils is mostly still there for backward support.
        setuptools is what smart people use,
        but it is not shipped with python :(
        """
        lines = []
        setup_type_line = '{name}_SETUP_TYPE = {method}\n'.format(
            name=self.mk_name,
            method=self.setup_metadata['method'])
        lines.append(setup_type_line)
        return lines

    def __create_mk_license(self):
        """
        Create the lines referring to the package's license informations of the
        <package_name>.mk file

        The license is found using the metadata from pypi.
        In the metadata, the license can be found either with standard names in
        the classifiers part or with naming from the packager in the "License"
        part.

        From the classifiers, the license is "translated" according to
        buildroot standards if need be (i.e. from Apache Software License to
        Apache-2.0).

        From the License part, we cannot guess what formatting the packager
        used. Hence, it is likely to be incorrect. (i.e. Apache License 2.0
        instead of Apache-2.0).

        The license's files are found by searching the package for files named
        license or license.txt (case insensitive).
        If more than one license file is found, the user is asked to select
        which ones he wants to use.
        """
        license_dict = {
            'Apache Software License': 'Apache-2.0',
            'BSD License': 'BSD',
            'European Union Public Licence 1.0': 'EUPLv1.0',
            'European Union Public Licence 1.1': 'EUPLv1.1',
            "GNU General Public License": "GPL",
            "GNU General Public License v2": "GPLv2",
            "GNU General Public License v2 or later": "GPLv2+",
            "GNU General Public License v3": "GPLv3",
            "GNU General Public License v3 or later": "GPLv3+",
            "GNU Lesser General Public License v2": "LGPLv2.1",
            "GNU Lesser General Public License v2 or later": "LGPLv2.1+",
            "GNU Lesser General Public License v3": "LGPLv3",
            "GNU Lesser General Public License v3 or later": "LGPLv3+",
            "GNU Library or Lesser General Public License": "LGPLv2",
            "ISC License": "ISC",
            "MIT License": "MIT",
            "Mozilla Public License 1.0": "MPL-1.0",
            "Mozilla Public License 1.1": "MPL-1.1",
            "Mozilla Public License 2.0": "MPL-2.0",
            "Zope Public License": "ZPL"
            }
        regexp = re.compile('^License :* *.* *:+ (.*)( \(.*\))?$')
        classifiers_licenses = [regexp.sub(r"\1", lic)
                                for lic in self.metadata['info']['classifiers']
                                if regexp.match(lic)]
        licenses = map(lambda x: license_dict[x] if x in license_dict else x,
                       classifiers_licenses)
        lines = []
        if not len(licenses):
            print('WARNING: License has been set to "{license}". It is most'
                  ' likely wrong, please change it if need be'.format(
                      license=', '.join(licenses)))
            licenses = [self.metadata['info']['license']]
        license_line = '{name}_LICENSE = {license}\n'.format(
            name=self.mk_name,
            license=', '.join(licenses))
        lines.append(license_line)

        filenames = ['LICENCE', 'LICENSE', 'LICENSE.TXT', 'COPYING',
		     'COPYING.TXT']
        license_files = list(find_file_upper_case(filenames, self.tmp_extract))
        license_files = [license.replace(self.tmp_extract, '')[1:]
                         for license in license_files]
        if len(license_files) > 0:
            if len(license_files) > 1:
                print('More than one file found for license:',
                      ', '.join(license_files))
            license_files = [filename
                             for index, filename in enumerate(license_files)]
            license_file_line = ('{name}_LICENSE_FILES ='
                                 ' {files}\n'.format(
                                     name=self.mk_name,
                                     files=' '.join(license_files)))
            lines.append(license_file_line)
        else:
            print('WARNING: No license file found,'
                  ' please specify it manually afterwards')
            license_file_line = '# No license file found\n'

        return lines

    def __create_mk_requirements(self):
        """
        Create the lines referring to the dependencies of the of the
        <package_name>.mk file

        Keyword Arguments:
        pkg_name -- name of the package
        pkg_req -- dependencies of the package
        """
        lines = []
        dependencies_line = ('{name}_DEPENDENCIES ='
                             ' {reqs}\n'.format(
                                 name=self.mk_name,
                                 reqs=' '.join(self.pkg_req)))
        lines.append(dependencies_line)
        return lines

    def create_package_mk(self):
        """
        Create the lines corresponding to the <package_name>.mk file
        """
        pkg_mk = '{name}.mk'.format(name=self.buildroot_name)
        path_to_mk = os.path.join(self.pkg_dir, pkg_mk)
        print('Creating {file}...'.format(file=path_to_mk))
        lines = self.__create_mk_header()
        lines += self.__create_mk_download_info()
        lines += self.__create_mk_setup()
        lines += self.__create_mk_license()

        lines.append('\n')
        lines.append('$(eval $(python-package))')
        lines.append('\n')
        with open(path_to_mk, 'w') as mk_file:
            mk_file.writelines(lines)

    def create_hash_file(self):
        """
        Create the lines corresponding to the <package_name>.hash files
        """
        pkg_hash = '{name}.hash'.format(name=self.buildroot_name)
        path_to_hash = os.path.join(self.pkg_dir, pkg_hash)
        print('Creating {filename}...'.format(filename=path_to_hash))
        lines = []
        if self.used_url['md5_digest']:
            md5_comment = '# md5 from {url}, sha256 locally computed\n'.format(
		url=self.metadata_url)
            lines.append(md5_comment)
            hash_line = '{method}\t{digest}  {filename}\n'.format(
                method='md5',
                digest=self.used_url['md5_digest'],
                filename=self.filename)
            lines.append(hash_line)
        digest = hashlib.sha256(self.as_string).hexdigest()
        hash_line = '{method}\t{digest}  {filename}\n'.format(
            method='sha256',
            digest=digest,
            filename=self.filename)
        lines.append(hash_line)

        with open(path_to_hash, 'w') as hash_file:
            hash_file.writelines(lines)

    def create_config_in(self):
        """
        Creates the Config.in file of a package
        """
        path_to_config = os.path.join(self.pkg_dir, 'Config.in')
        print('Creating {file}...'.format(file=path_to_config))
        lines = []
        config_line = 'config BR2_PACKAGE_{name}\n'.format(
            name=self.mk_name)
        lines.append(config_line)

        bool_line = '\tbool "{name}"\n'.format(name=self.buildroot_name)
        lines.append(bool_line)
        if self.pkg_req:
            for dep in self.pkg_req:
                dep_line = '\tselect BR2_PACKAGE_{req} # runtime\n'.format(
                    req=dep.upper().replace('-', '_'))
                lines.append(dep_line)

        lines.append('\thelp\n')

        help_lines = textwrap.wrap(self.metadata['info']['summary'],
                                   initial_indent='\t  ',
                                   subsequent_indent='\t  ')

        # make sure a help text is terminated with a full stop
        if help_lines[-1][-1] != '.':
            help_lines[-1] += '.'

        # \t + two spaces is 3 char long
        help_lines.append('')
        help_lines.append('\t  ' + self.metadata['info']['home_page'])
        help_lines = map(lambda x: x + '\n', help_lines)
        lines += help_lines

        with open(path_to_config, 'w') as config_file:
            config_file.writelines(lines)


def main():
    # Building the parser
    parser = argparse.ArgumentParser(
        description="Creates buildroot packages from the metadata of "
                    "an existing PyPI packages and include it "
                    "in menuconfig")
    parser.add_argument("packages",
                        help="list of packages to be created",
                        nargs='+')
    parser.add_argument("-o", "--output",
                        help="""
                        Output directory for packages.
                        Default is ./package
                        """,
                        default='./package')

    args = parser.parse_args()
    packages = list(set(args.packages))

    # tmp_path is where we'll extract the files later
    tmp_prefix = 'scanpypi-'
    pkg_folder = args.output
    tmp_path = tempfile.mkdtemp(prefix=tmp_prefix)
    try:
        for real_pkg_name in packages:
            package = BuildrootPackage(real_pkg_name, pkg_folder)
            print('buildroot package name for {}:'.format(package.real_name),
                  package.buildroot_name)
            # First we download the package
            # Most of the info we need can only be found inside the package
            print('Package:', package.buildroot_name)
            print('Fetching package', package.real_name)
            try:
                package.fetch_package_info()
            except (urllib2.URLError, urllib2.HTTPError):
                continue
            if package.metadata_name.lower() == 'setuptools':
                # setuptools imports itself, that does not work very well
                # with the monkey path at the begining
                print('Error: setuptools cannot be built using scanPyPI')
                continue

            try:
                package.download_package()
            except urllib2.HTTPError as error:
                print('Error: {code} {reason}'.format(code=error.code,
                                                      reason=error.reason))
                print('Error downloading package :', package.buildroot_name)
                print()
                continue

            # extract the tarball
            try:
                package.extract_package(tmp_path)
            except (tarfile.ReadError, zipfile.BadZipfile):
                print('Error extracting package {}'.format(package.real_name))
                print()
                continue

            # Loading the package install info from the package
            try:
                package.load_setup()
            except ImportError as err:
                if 'buildutils' in err.message:
                    print('This package needs buildutils')
                else:
                    raise
                continue
            except AttributeError:
                print('Error: Could not install package {pkg}'.format(
                    pkg=package.real_name))
                continue

            # Package requirement are an argument of the setup function
            req_not_found = package.get_requirements(pkg_folder)
            req_not_found = req_not_found.difference(packages)

            packages += req_not_found
            if req_not_found:
                print('Added packages \'{pkgs}\' as dependencies of {pkg}'
                      .format(pkgs=", ".join(req_not_found),
                              pkg=package.buildroot_name))
            print('Checking if package {name} already exists...'.format(
                name=package.pkg_dir))
            try:
                os.makedirs(package.pkg_dir)
            except OSError as exception:
                if exception.errno != errno.EEXIST:
                    print("ERROR: ", exception.message, file=sys.stderr)
                    continue
                print('Error: Package {name} already exists'
                      .format(name=package.pkg_dir))
                del_pkg = raw_input(
                    'Do you want to delete existing package ? [y/N]')
                if del_pkg.lower() == 'y':
                    shutil.rmtree(package.pkg_dir)
                    os.makedirs(package.pkg_dir)
                else:
                    continue
            package.create_package_mk()

            package.create_hash_file()

            package.create_config_in()
            print()
            # printing an empty line for visual confort
    finally:
        shutil.rmtree(tmp_path)

if __name__ == "__main__":
    main()