From 28acf97554a2d445806b93140c757142e2cd2062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Wed, 1 Nov 2023 10:05:04 +0100 Subject: [PATCH] Support GitHub public repo source --- src/smp_importer/app.py | 63 ++++++++-- src/smp_importer/static/script.js | 140 ++++++++++++++--------- src/smp_importer/static/style.css | 37 ++++++ src/smp_importer/templates/index.html.j2 | 61 +++++++--- 4 files changed, 224 insertions(+), 77 deletions(-) diff --git a/src/smp_importer/app.py b/src/smp_importer/app.py index 32c7a59..21d3563 100644 --- a/src/smp_importer/app.py +++ b/src/smp_importer/app.py @@ -1,4 +1,6 @@ +import base64 import logging +import mimetypes import os import pathlib @@ -15,6 +17,7 @@ ROOT_DIR = pathlib.Path(__file__).parent STATIC_DIR = ROOT_DIR / 'static' TEMPLATES_DIR = ROOT_DIR / 'templates' +GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN', None) LOG = logging.getLogger(__name__) @@ -38,6 +41,12 @@ class BodyFile(pydantic.BaseModel): bytesize: int +class BodyGitHubPublic(pydantic.BaseModel): + owner: str + repo: str + file: str + + @app.get('/', response_class=fastapi.responses.HTMLResponse) async def get_index(request: fastapi.Request): return templates.TemplateResponse( @@ -51,15 +60,15 @@ async def get_index(request: fastapi.Request): @app.post('/api/import-file', response_class=fastapi.responses.JSONResponse) -async def api_import_from_file(body_file: BodyFile): +async def api_import_from_file(body: BodyFile): try: result = process( - content=body_file.contents, - content_type=body_file.type, + content=body.contents, + content_type=body.type, ) return fastapi.responses.JSONResponse(content={ 'sourcce': 'file', - 'name': body_file.name, + 'name': body.name, 'actions': result['actions'], }) except Exception as e: @@ -68,12 +77,28 @@ async def api_import_from_file(body_file: BodyFile): @app.post('/api/import-url', response_class=fastapi.responses.JSONResponse) -async def api_import_from_url(body_url: BodyURL): +async def api_import_from_url(body: BodyURL): try: - result = await fetch_from_url(body_url.url) + result = await fetch_from_url(body.url) return fastapi.responses.JSONResponse(content={ 'source': 'url', - 'url': body_url.url, + 'url': body.url, + 'actions': result['actions'], + }) + except Exception as e: + LOG.error(f'Error appeared: {str(e)}', exc_info=e) + raise fastapi.HTTPException(status_code=500) + + +@app.post('/api/import-github-public', response_class=fastapi.responses.JSONResponse) +async def api_import_from_github_public(body: BodyGitHubPublic): + try: + result = await fetch_from_github_public(body.owner, body.repo, body.file) + return fastapi.responses.JSONResponse(content={ + 'source': 'github', + 'owner': body.owner, + 'repo': body.repo, + 'file': body.file, 'actions': result['actions'], }) except Exception as e: @@ -91,3 +116,27 @@ async def fetch_from_url(url: str) -> dict: content=r.content.decode(encoding=r.charset_encoding or 'utf-8'), content_type=r.headers.get('content-type'), ) + + +async def fetch_from_github_public(owner: str, repo: str, file: str) -> dict: + headers = { + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + } + if GITHUB_TOKEN is not None: + headers['Authorization'] = f'Bearer {GITHUB_TOKEN}' + + async with httpx.AsyncClient() as client: + r = await client.get( + url=f'https://api.github.com/repos/{owner}/{repo}/contents/{file}', + headers=headers + ) + r.raise_for_status() + + data = r.json() + if data.get('encoding') == 'base64': + return process( + content=base64.b64decode(data['content']).decode('utf-8'), + content_type=mimetypes.guess_type(data['url'])[0] or 'text/plain', + ) + raise RuntimeError('Unexpected response from GitHub') diff --git a/src/smp_importer/static/script.js b/src/smp_importer/static/script.js index e41487d..2ce14ee 100644 --- a/src/smp_importer/static/script.js +++ b/src/smp_importer/static/script.js @@ -4,7 +4,7 @@ importer .init({ useWizardStyles: true, windowSize: { - width: 300, + width: 600, height: 500, }, }) @@ -12,63 +12,21 @@ importer jQuery('#file-input').on('input', function (e) { const files = e.target.files console.log(files) - const file = files[0] - - const reader = new FileReader() - reader.addEventListener('load', (event) => { - let data = '' - try { - data = event.target.result - } catch (error) { - alert('Failed to load file') - } + loadFile(files[0]) + }) - jQuery.ajax({ - type: 'POST', - url: `/api/import-file`, - data: JSON.stringify({ - 'contents': data, - 'type': file.type, - 'name': file.name, - 'bytesize': file.size, - }), - contentType: "application/json; charset=utf-8", - traditional: true, - success: function (result) { - doImport(result.actions) - }, - error: function (result) { - console.log(result) - alert('failed') - } - }) - }) - reader.readAsText(file) + jQuery('#btn-github-load').on('click', function () { + const owner = jQuery('#github-owner-input').val() + const repo = jQuery('#github-repo-input').val() + const file = jQuery('#github-file-input').val() + console.log(`${owner}/${repo}:/${file}`) + loadGitHubPublic(owner, repo, file) }) - jQuery('#btn-load').on('click', function () { + jQuery('#btn-url-load').on('click', function () { const url = jQuery('#url-input').val() console.log(url) - - if (!isValidUrl(url)) { - alert('Invalid URL!') - jQuery('#url-input').val('') - } else { - jQuery.ajax({ - type: 'POST', - url: `/api/import-url`, - data: JSON.stringify({'url': url}), - contentType: "application/json; charset=utf-8", - traditional: true, - success: function (result) { - doImport(result.actions) - }, - error: function (result) { - console.log(result) - alert('failed') - } - }) - } + loadUrl(url) }) }) .catch(error => { @@ -129,3 +87,79 @@ function doImport(actions) { importer.send() } +function loadUrl(url) { + if (!isValidUrl(url)) { + alert('Invalid URL!') + jQuery('#url-input').val('') + } else { + jQuery.ajax({ + type: 'POST', + url: `/api/import-url`, + data: JSON.stringify({'url': url}), + contentType: "application/json; charset=utf-8", + traditional: true, + success: function (result) { + doImport(result.actions) + }, + error: function (result) { + console.log(result) + alert('failed') + } + }) + } +} + +function loadFile(file) { + const reader = new FileReader() + reader.addEventListener('load', (event) => { + let data = '' + try { + data = event.target.result + } catch (error) { + alert('Failed to load file') + } + + jQuery.ajax({ + type: 'POST', + url: `/api/import-file`, + data: JSON.stringify({ + 'contents': data, + 'type': file.type, + 'name': file.name, + 'bytesize': file.size, + }), + contentType: "application/json; charset=utf-8", + traditional: true, + success: function (result) { + doImport(result.actions) + }, + error: function (result) { + console.log(result) + alert('failed') + } + }) + }) + reader.readAsText(file) +} + +function loadGitHubPublic(owner, repo, file) { + jQuery.ajax({ + type: 'POST', + url: `/api/import-github-public`, + data: JSON.stringify({ + 'owner': owner, + 'repo': repo, + 'file': file, + }), + contentType: "application/json; charset=utf-8", + traditional: true, + success: function (result) { + doImport(result.actions) + }, + error: function (result) { + console.log(result) + alert('failed') + } + }) +} + diff --git a/src/smp_importer/static/style.css b/src/smp_importer/static/style.css index e69de29..360626d 100644 --- a/src/smp_importer/static/style.css +++ b/src/smp_importer/static/style.css @@ -0,0 +1,37 @@ +html, body { + width: 100%; +} + +.container { + padding: 1em; +} + +.hint { + font-size: 80%; + color: rgb(128, 128, 128); +} + +.input-group { + margin: 0 !important; +} + +.hide { + display: none; +} + +.debug { + display: none; +} + +.debug .form-check { + margin: 0; + padding: 0; +} + +.debug .form-check .form-check-input { + margin-left: 0; +} + +.debug .form-check .form-check-label { + margin-left: 1em; +} diff --git a/src/smp_importer/templates/index.html.j2 b/src/smp_importer/templates/index.html.j2 index 704c654..368e815 100644 --- a/src/smp_importer/templates/index.html.j2 +++ b/src/smp_importer/templates/index.html.j2 @@ -2,13 +2,13 @@ - SIP Importer + SMP Importer - {% if development %} @@ -36,6 +38,17 @@
+ {% if debug %} +
+ +
+ {% endif %} +

SMP Importer

@@ -44,7 +57,6 @@
-

@@ -53,16 +65,31 @@
- -

- +
+ + +
-
- - 

+            

+                
Public GitHub repository

 
+                

+                    
+                    

+                        

+                            
+                            /
+                            
+                            :/
+                            
+                            
+                        

+                        
This works only for public GitHub repositories...

+                    

+                

+            

         
     
 
\ No newline at end of file