WIP: Track feedstocks

libcflib/feedstockutils.xsh

@@ -0,0 +1,47 @@
+import re
+
+from libcflib.tools import parse_meta_yaml
+
+sel_pat = re.compile(".+?\s*(#.*)?\[[^\[\]]+\](?(1)[^\(\)]*)$")
+ARCHS = ["linux-32", "linux-64", "linux-arm", "osx-64", "win-32", "win-64"]
+
+
+def has_selectors(meta_yaml):
+    for line in meta_yaml.splitlines():
+        line = line.rstrip()
+        if line.lstrip().startswith("#"):
+            continue
+        if sel_pat.match(line):
+            return True
+    return False
+
+
+def get_packages_from_recipe(recipe):
+    outputs = recipe.get("outputs", recipe["package"])
+    packages = set()
+    for package in outputs:
+        packages.add(package)
+    return packages
+
+
+def extract_packages_from_feedstock(feedstock, arch=None):
+    packages = set()
+    if not arch and has_selectors(feedstock.meta_yaml):
+        for a in ARCHS:
+            packages.update(extract_packages_from_feedstock(feedstock, arch=a))
+    elif arch:
+        return get_packages_from_recipe(parse_meta_yaml(feedstock.meta_yaml, arch=arch))
+    else:
+        return get_packages_from_recipe(parse_meta_yaml(feedstock.meta_yaml))
+
+
+def get_feedstock_names():
+    git init feedstock
+    with indir(feedstocks):
+        git remote add origin git@github.com:conda-forge/feedstocks.git
+        git config core.sparsecheckout true
+        echo "feedstocks/" >> .git/info/sparse-checkout
+        git pull --depth=1 origin master
+        feedstocks = $(ls feedstocks/).splitlines()
+    rm -rf feedstocks
+    return feedstocks

libcflib/harvester.py

@@ -23,23 +23,26 @@ def filter_file(filename):
     else:
         return True
 
+
 # There are some meta.yaml files that were written weirdly and have some strange tag
-# information in them.  This is not parsable by safe loader, so we add some more 
+# information in them.  This is not parsable by safe loader, so we add some more
 # fallback loading
 def yaml_construct_fallback(loader, node):
     return None
 
+
 ruamel_yaml.add_constructor(
-    'tag:yaml.org,2002:python/object/apply:builtins.getattr', 
+    "tag:yaml.org,2002:python/object/apply:builtins.getattr",
     yaml_construct_fallback,
     constructor=ruamel_yaml.SafeConstructor,
 )
 ruamel_yaml.add_constructor(
-    'tag:yaml.org,2002:python/object:__builtin__.instancemethod',
+    "tag:yaml.org,2002:python/object:__builtin__.instancemethod",
     yaml_construct_fallback,
     constructor=ruamel_yaml.SafeConstructor,
 )
 
+
 def harvest(io_like):
     tf = tarfile.open(fileobj=io_like, mode="r:bz2")
 

libcflib/preloader.py

@@ -110,7 +110,7 @@ def reap_package(root_path, package, dst_path, src_url, progress_callback=None):
 
 def reap(path, known_bad_packages=()):
     sorted_files = list(diff(path))
-    print(f"TOTAL OUTSTANDING ARTIFACTS: {len(sorted_files)}")    
+    print(f"TOTAL OUTSTANDING ARTIFACTS: {len(sorted_files)}")
     sorted_files = sorted_files[:500]
     progress = tqdm.tqdm(total=len(sorted_files))
     with ThreadPoolExecutor(max_workers=20) as pool:

libcflib/rest/request_handler.py

@@ -57,7 +57,7 @@ def set_default_headers(self):
         self.set_header("Content-Type", "application/json")
         self.set_header("Access-Control-Allow-Origin", "*")
         self.set_header("Access-Control-Allow-Headers", "x-requested-with")
-        self.set_header('Access-Control-Allow-Methods', 'GET, OPTIONS')
+        self.set_header("Access-Control-Allow-Methods", "GET, OPTIONS")
 
     def write(self, chunk):
         """Writes the given chunk to the output buffer. This overrides (and almost

libcflib/tools.xsh

@@ -81,7 +81,7 @@ def render_meta_yaml(text):
     return content
 
 
-def parse_meta_yaml(text):
+def parse_meta_yaml(text, arch=None):
     """Parse the meta.yaml.
 
     Parameters
@@ -96,9 +96,12 @@ def parse_meta_yaml(text):
 
     """
 
+    config = Config()
+    if arch:
+        config.host_subdir = arch
     try:
         content = render_meta_yaml(text)
-        return parse(content, Config())
+        return parse(content, config)
     except:
         return {}
 
@@ -109,4 +112,4 @@ def indir(d):
     old_d = os.getcwd()
     ![cd @(d)]
     yield
-    ![cd @(old_d)]
+    ![cd @(old_d)]