Merge pull request #6 from vgmstream/master

Update

.github/changelog.py

@@ -46,15 +46,16 @@ def convert_git(stdout):
 def load_git():
     if not USE_GIT:
         raise ValueError("git disabled")
-        
+
     args = ['git', 'describe', '--tags', '--abbrev=0']
     proc = subprocess.run(args, capture_output=True)
     if proc.returncode != 0:
         raise ValueError("git exception")
     latest_tag = proc.stdout.decode('utf-8').strip()
-
+
+    # no apparent portable way to get a utc timezone in date=format
     #args = ['git','--no-pager','log', '--merges', '--date=format:"%Y-%m-%d %H:%M:%S"',  '--max-count', str(GIT_MAX_MERGES) ]
-    args = ['git','--no-pager','log', '--merges', '--date=format:"%Y-%m-%d %H:%M:%S"', '%s..HEAD' % (latest_tag)]
+    args = ['git','--no-pager','log', '--merges', '--date=format:"%Y-%m-%d %H:%M:%S %z"', '%s..HEAD' % (latest_tag)]
     proc = subprocess.run(args, capture_output=True)
     if proc.returncode != 0:
         raise ValueError("git exception")
@@ -160,7 +161,7 @@ def get_lines(short_log=False):
     if short_log:
         lines = []
     else:
-        curr_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        curr_date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %z")
         lines = [
             '### CHANGELOG',
             '(latest changes from previous release, generated on %s)' % (curr_date),

.github/formats-info.py

@@ -0,0 +1,369 @@
+# Generates formats-info.md based on src/meta code, to update FORMATS.md.
+# A bit crummy and all but whatevs.
+
+import glob, os, re
+
+#TODO fix some "ext + ext" exts
+#TODO improve dsp/ubi/ea formats
+#TODO detect subfile/init_x calls that aren't VGMSTREAM * init_
+#TODO detect MPEG_x, OPUS_x subtypes
+#TODO ignore some check extensions?
+#TODO maybe include common idstrings (first is_id32be() / is_id32le / get_id32be / get_id32be) 
+#     > after first valid init?
+
+META_SRC = '../**/meta/*.c'
+FORMAT_SRC = '../**/formats.c'
+SORT_SRC = '../**/vgmstream.c'
+FORMAT_DOC = '../**/FORMATS.md'
+DUMP_DOC = 'formats-info.md'
+IS_SORT = True          #sorts metas lines based on LIST_SRC
+IS_PRINT_DOC = False    #echo list
+IS_DUMP_DOC = False     #creates DUMP_DOC
+IS_UPDATE_DOC = True    #updates FORMAT_DOC
+
+#VGMSTREAM * init_vgmstream_9tav(STREAMFILE *sf) #allow static since it's used in some cases
+RE_INIT = re.compile(r"VGMSTREAM[ ]*\*[ ]*init_vgmstream_([A-Za-z0-9_]+)\(.+\)")
+RE_INIT_STATIC = re.compile(r"static VGMSTREAM[ ]*\*[ ]*init_vgmstream_([A-Za-z0-9_]+)\(.+\)")
+
+RE_CODING = re.compile(r"coding_([A-Z0-9][A-Za-z0-9_]+)")
+
+RE_META = re.compile(r"meta_([A-Z0-9][A-Za-z0-9_]+)")
+
+#sf_dat = open_streamfile_by_ext(sf, "dat");
+RE_COMPANION_EXT = re.compile(r"open_streamfile_by_ext\(.+,[ ]*\"(.+)\"[ ]*\)")
+
+# a = init_...
+RE_SUBFILES = re.compile(r"^(?!VGMSTREAM).*init_vgmstream_([A-Za-z0-9_]+)[(;]")
+RE_COMPANION_FILE = re.compile(r"open_streamfile_by_filename\(.+,[ ]*(.+)?\)")
+
+# if (!check_extensions(sf,"..."))
+# if (check_extensions(sf,"...")) { ... } else goto fail;
+RE_NEW_EXTCHECK = re.compile(r"\([ ]*[!]*check_extensions\([ ]*.+?,[ ]*\"(.+)\"[ ]*\)")
+#if (strcasecmp("bg00",filename_extension(filename))) goto fail;
+RE_OLD_EXTCHECK = re.compile(r"if[ ]*\(strcasecmp\([ ]*\"(.+)\"[ ]*,filename_extension")
+
+# formats.c extract
+RE_FORMATS_META = re.compile(r"{meta_([A-Z0-9][A-Za-z0-9_]+)[ ]*,[ ]* \"(.+)\"[ ]*}")
+
+# vgmstream.c meta call extract
+RE_SORT_INIT = re.compile(r"[ ]+init_vgmstream_([A-Za-z0-9_]+),")
+
+FILES_SKIP = [
+    'txth.c','txtp.c','genh.c', 
+    'silence.c', 'mp4_faac.c', 'deblock_streamfile.c', 
+    'ps_headerless.c', 'zwdsp.c',
+]
+
+EXT_RENAMES = {'...': '(any)', '': '(extensionless)'}
+INIT_IGNORES = ['default', 'main']
+META_IGNORES = ['meta_type', 'meta_t']
+CODING_IGNORES = ['coding_type', 'coding_t', 'FFmpeg', 'SILENCE']
+SUBFILES_IGNORES = ['subkey', 'silence', 'silence_container']
+
+# detect better various codecs that use a generic coding_XXX
+INIT_CODECS = {
+    re.compile(r'init_ffmpeg_aac\('): 'AAC',
+    re.compile(r'init_ffmpeg_xwma\('): 'XWMA',
+    re.compile(r'init_ffmpeg_atrac3plus_.+\('): 'ATRAC3PLUS',
+    re.compile(r'init_ffmpeg_atrac3_.+\('): 'ATRAC3',
+    re.compile(r'init_ffmpeg_xma_.+\('): 'XMA',
+    re.compile(r'init_ffmpeg_xma1_.+\('): 'XMA1',
+    re.compile(r'init_ffmpeg_xma2_.+\('): 'XMA2',
+    re.compile(r'init_mpeg_custom\('): 'MPEG',
+    re.compile(r'init_ffmpeg_mp4_.+\('): 'MP4/AAC',
+    re.compile(r'init_ffmpeg_offset\('): 'FFmpeg(various)',
+    re.compile(r'init_ffmpeg_header_offset\('): 'FFmpeg(various)',
+    re.compile(r'init_ffmpeg_header_offset_subsong\('): 'FFmpeg(various)',
+    re.compile(r'init_ffmpeg_.+?_opus.*\('): 'Opus',
+    re.compile(r'init_vgmstream_ogg_vorbis_config\('): 'OGG',
+}
+
+class MetaInit:
+    def __init__(self, name):
+        self.name = name
+        self.subfiles = []
+        self.exts = []
+        self.companions = []
+        #self.metas = []
+
+    def is_used(self):
+        return self.subfiles or self.exts or self.companions
+
+class MetaFile:
+    def __init__(self, file):
+        self.fileinfo = os.path.basename(file)
+        self.order = 9999
+        # divide each file into sub-metas since some .c have a bunch of them
+        self.curr = MetaInit('default') #shouldn't be used but...
+        self.inits = [self.curr]
+        self.metas = []
+        self.codings = []
+
+    def add_inits(self, elems):
+        if len(elems) > 1:
+            raise ValueError("multiinit?")
+
+        for elem in elems:
+            self.curr = MetaInit(elem)
+            self.inits.append(self.curr)
+
+    def _add(self, new_items, items, renames={}):
+        if not new_items:
+            return
+
+        for new_item in new_items:
+            if new_item in renames:
+                new_item = renames[new_item]
+            if new_item not in items:
+                items.append(new_item)
+
+    #def add_inits(self, elems):
+    #    self._add(elems, self.inits)
+
+    def add_codings(self, codings):
+        self._add(codings, self.codings)
+
+    def add_exts(self, exts, renames={}):
+        self._add(exts, self.curr.exts, renames)
+
+    def add_metas(self, metas):
+        self._add(metas, self.metas)
+
+    def add_companions(self, elems):
+        self._add(elems, self.curr.companions)
+
+    def add_subfiles(self, elems):
+        self._add(elems, self.curr.subfiles)
+
+class App:
+    def __init__(self):
+        self.infos = []
+        self.desc_metas = {}
+        self.lines = []
+
+    def extract_extensions(self, line):
+
+        exts = RE_NEW_EXTCHECK.findall(line)
+        if not exts:
+            exts = RE_OLD_EXTCHECK.findall(line)
+        if not exts:
+            return
+        exts_str = ','.join(exts)
+        return exts_str.split(',')
+
+    def extract_regex(self, line, regex, ignores=[]):
+        items = []
+        results = regex.findall(line)
+        for result in results:
+            if result in ignores:
+                continue
+            items.append(result)
+        return items
+
+    def extract_codecs(self, line):
+        for regex, codec in INIT_CODECS.items():
+            if regex.search(line):
+                return [codec]
+
+    def parse_files(self):
+        infos = self.infos
+
+        files = glob.glob(META_SRC, recursive=True)
+        for file in files:
+            info = MetaFile(file)
+            with open(file, 'r', encoding='utf-8') as f:
+                internals = []
+                for line in f:
+                    # use to ignore subcalls to internal functions
+                    items = self.extract_regex(line, RE_INIT_STATIC)
+                    if items:
+                        internals += items
+
+                    items = self.extract_regex(line, RE_INIT)
+                    info.add_inits(items)
+
+                    items = self.extract_extensions(line)
+                    info.add_exts(items, EXT_RENAMES)
+
+                    items = self.extract_regex(line, RE_META, META_IGNORES)
+                    info.add_metas(items)
+
+                    items = self.extract_regex(line, RE_CODING, CODING_IGNORES)
+                    info.add_codings(items)
+
+                    items = self.extract_regex(line, RE_COMPANION_EXT)
+                    info.add_companions(items)
+
+                    items = self.extract_regex(line, RE_COMPANION_FILE)
+                    if items:
+                        info.add_companions(['(external)'])
+
+                    items = self.extract_regex(line, RE_SUBFILES, SUBFILES_IGNORES)
+                    if not any(x in internals for x in items):
+                        info.add_subfiles(items)
+
+                    items = self.extract_codecs(line)
+                    info.add_codings(items)
+
+            infos.append(info)
+
+    def parse_formats(self):
+        desc_metas = self.desc_metas
+
+        files = glob.glob(FORMAT_SRC, recursive=True)
+        for file in files:
+            with open(file, 'r', encoding='utf-8') as f:
+                for line in f:
+                    items = self.extract_regex(line, RE_FORMATS_META)
+                    if not items or len(items[0]) != 2:
+                        continue
+                    meta, desc = items[0]
+                    desc_metas[meta] = desc
+
+    def sort_files(self):
+        inits_order = {}
+
+        # get process list order
+        files = glob.glob(SORT_SRC, recursive=True)
+        order = 0
+        for file in files:
+            with open(file, 'r', encoding='utf-8') as f:
+                for line in f:
+                    items = self.extract_regex(line, RE_SORT_INIT)
+                    if not items or len(items) != 1:
+                        continue
+                    init = items[0]
+                    inits_order[init] = order
+                    order += 1
+
+        # update (could be done on first pass or maybe in sort() but it's a bit specialized)
+        infos = self.infos
+        for info in infos:
+            for init in info.inits:
+                order = inits_order.get(init.name)
+                #print(init, order)
+                if order is not None and info.order > order:
+                    info.order = order
+
+        infos.sort(key=lambda x: x.order) #, reverse=True
+
+
+    def prepare_results(self):
+        desc_metas = self.desc_metas
+        infos = self.infos
+
+
+        lines = self.lines
+        for info in infos:
+            #info.sort()
+            if info.fileinfo in FILES_SKIP:
+                continue
+
+            fileinfo = '- **%s**' % (info.fileinfo)
+            #if info.metas:
+            #    metas = ' '.join(info.metas)
+            #    fileinfo += " (%s)" % (metas)
+            lines.append(fileinfo)
+
+            if info.metas:
+                for meta in info.metas:
+                    desc = desc_metas.get(meta, '???')
+                    metainfo = "%s [*%s*]" % (desc, meta)
+                    lines.append('  - ' + metainfo)
+            else:
+                lines.append('  - (container)')
+
+            for init in info.inits:
+                if not init.is_used():
+                    continue
+                if INIT_IGNORES and init.name in INIT_IGNORES:
+                    continue
+
+                initinfo = '*%s*' % (init.name)
+
+                infoexts = ''
+                if init.exts:
+                    infoexts = '.' + ' .'.join(init.exts)
+                if init.companions:
+                    if not init.exts:
+                        infoexts = '(base)'
+                    infoexts += ' + .' + ' .'.join(init.companions)
+                if infoexts:
+                    #lines.append(infoexts)
+                    initinfo += ": `%s`" % infoexts
+                lines.append('  - ' + initinfo)
+
+                if init.subfiles:
+                    subfiles = 'Subfiles: *%s*' % ' '.join(init.subfiles)
+                    lines.append('    - ' + subfiles)
+
+            if info.codings:
+                codings = 'Codecs: ' + ' '.join(info.codings)
+                lines.append('  - ' + codings)
+
+            #lines.append('')
+
+
+    def update_doc(self):
+        new_lines = self.lines
+
+        files = glob.glob(FORMAT_DOC, recursive=True)
+        if not files:
+            print("format doc (%s) not found" % FORMAT_DOC)
+            return
+
+        doc_lines = []
+        file = files[0]
+        with open(file, 'r', encoding='utf-8') as f:
+            is_updating = False
+            for line in f:
+                line = line.strip()
+
+                # find list section, add all current lines and ignore old ones until next section
+                if line.startswith('### List'):
+                    is_updating = True
+                    doc_lines.append(line)
+                    doc_lines += new_lines
+                    doc_lines.append('')
+                    continue
+
+                # next section
+                if is_updating and line.startswith('#'):
+                    is_updating = False
+
+                if not is_updating:
+                    doc_lines.append(line)
+
+        with open(file, 'w', encoding='utf-8', newline='\n') as f:
+            f.write('\n'.join(doc_lines))
+
+
+    def dump_doc(self):
+        lines = self.lines
+
+        with open(DUMP_DOC, 'w', encoding='utf-8') as f:
+            f.write('\n'.join(lines))
+
+
+    def print_doc(self):
+        lines = self.lines
+        print('\n'.join(lines))
+
+
+    def process(self):
+        self.parse_files()
+        self.parse_formats()
+        if IS_SORT:
+            self.sort_files()
+
+        self.prepare_results()
+
+        if IS_PRINT_DOC:
+            self.print_doc()
+        if IS_DUMP_DOC:
+            self.dump_doc()
+        if IS_UPDATE_DOC:
+            self.update_doc()
+
+
+App().process()