from genericpath import isfile
|
import importlib
|
import json
|
from posixpath import join
|
import time
|
import requests
|
import os
|
from os import listdir, path
|
|
from app import models, pending_actions
|
from app.security import path_traversal_check
|
from app.plugins.views import TaskView
|
from app.plugins.worker import run_function_async, task
|
from app.plugins import get_current_plugin
|
from app.plugins import GlobalDataStore, get_site_settings, signals as plugin_signals
|
|
from coreplugins.dronedb.ddb import DEFAULT_HUB_URL, DroneDB, parse_url, verify_url
|
|
from django.dispatch import receiver
|
|
from worker.celery import app
|
from rest_framework.response import Response
|
from rest_framework import status
|
|
VALID_IMAGE_EXTENSIONS = ['.tiff', '.tif', '.png', '.jpeg', '.jpg']
|
|
def is_valid(file):
|
_, file_extension = path.splitext(file)
|
return file_extension.lower() in VALID_IMAGE_EXTENSIONS or file == 'gcp_list.txt' or file == 'geo.txt'
|
|
def get_settings(request):
|
ds = get_current_plugin().get_user_data_store(request.user)
|
|
registry_url = ds.get_string('registry_url') or DEFAULT_HUB_URL
|
username = ds.get_string('username') or None
|
password = ds.get_string('password') or None
|
token = ds.get_string('token') or None
|
|
return registry_url, username, password, token
|
|
|
def update_token(request, token):
|
ds = get_current_plugin().get_user_data_store(request.user)
|
ds.set_string('token', token)
|
|
def get_ddb(request):
|
registry_url, username, password, token = get_settings(request)
|
|
if registry_url == None or username == None or password == None:
|
raise ValueError('Credentials must be set.')
|
|
return DroneDB(registry_url, username, password, token, lambda token: update_token(request, token))
|
|
def to_web_protocol(registry_url):
|
return registry_url.replace('ddb+unsafe://', 'http://').replace('ddb://', 'https://').rstrip('/')
|
|
class CheckCredentialsTaskView(TaskView):
|
def post(self, request):
|
|
# Read form data
|
hub_url = request.data.get('hubUrl', None)
|
username = request.data.get('userName', None)
|
password = request.data.get('password', None)
|
|
# Make sure both values are set
|
if hub_url == None or username == None or password == None:
|
return Response({'error': 'All fields must be set.'}, status=status.HTTP_400_BAD_REQUEST)
|
|
try:
|
|
ddb = DroneDB(hub_url, username, password)
|
|
return Response({'success': ddb.login()}, status=status.HTTP_200_OK)
|
|
except(Exception) as e:
|
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
|
|
class OrganizationsTaskView(TaskView):
|
def get(self, request):
|
|
try:
|
|
ddb = get_ddb(request)
|
|
orgs = ddb.get_organizations()
|
|
return Response(orgs, status=status.HTTP_200_OK)
|
|
except Exception as e:
|
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
|
|
|
class DatasetsTaskView(TaskView):
|
def get(self, request, org=None):
|
|
if org == None:
|
return Response({'error': 'Organization must be set.'}, status=status.HTTP_400_BAD_REQUEST)
|
|
try:
|
|
ddb = get_ddb(request)
|
|
dss = ddb.get_datasets(org)
|
|
return Response(dss, status=status.HTTP_200_OK)
|
|
except Exception as e:
|
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
|
|
class FoldersTaskView(TaskView):
|
def get(self, request, org=None, ds=None):
|
|
if org == None or ds == None:
|
return Response({'error': 'Organization and dataset must be set.'}, status=status.HTTP_400_BAD_REQUEST)
|
|
try:
|
|
ddb = get_ddb(request)
|
|
folders = ddb.get_folders(org, ds)
|
|
return Response(folders, status=status.HTTP_200_OK)
|
|
except Exception as e:
|
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
|
|
class VerifyUrlTaskView(TaskView):
|
def post(self, request):
|
|
# Read form data
|
url = request.data.get('url', None)
|
|
if url == None:
|
return Response({'error': 'Url must be set.'}, status=status.HTTP_400_BAD_REQUEST)
|
|
_, username, password, _ = get_settings(request)
|
|
try:
|
|
result, org, ds, folder, count, size = verify_url(url, username, password).values()
|
|
if (not result):
|
return Response({'error': 'Invalid url.'}, status=status.HTTP_400_BAD_REQUEST)
|
|
return Response({'count': count, 'success': result, 'ds' : ds, 'org': org, 'folder': folder or None, 'size': size}
|
if org else {'success': False}, status=status.HTTP_200_OK)
|
|
except Exception as e:
|
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
|
|
class InfoTaskView(TaskView):
|
def get(self, request):
|
|
registry_url, username, _, _ = get_settings(request)
|
|
return Response({ 'hubUrl': registry_url, 'username': username }, status=status.HTTP_200_OK)
|
|
|
class ImportDatasetTaskView(TaskView):
|
def post(self, request, project_pk=None, pk=None):
|
|
task = self.get_and_check_task(request, pk)
|
|
# Read form data
|
ddb_url = request.data.get('ddb_url', None)
|
|
if ddb_url == None:
|
return Response({'error': 'DroneDB url must be set.'}, status=status.HTTP_400_BAD_REQUEST)
|
|
registry_url, orgSlug, dsSlug, folder = parse_url(ddb_url).values()
|
|
_, username, password, token = get_settings(request)
|
ddb = DroneDB(registry_url, username, password, token, lambda token: update_token(request, token))
|
|
# Get the files from the folder
|
rawfiles = ddb.get_files_list(orgSlug, dsSlug, folder)
|
files = [file for file in rawfiles if is_valid(file['path'])]
|
|
# Verify that the folder url is valid
|
if len(files) == 0:
|
return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST)
|
|
# Update the task with the new information
|
task.console += "Importing {} images...\n".format(len(files))
|
task.images_count = len(files)
|
task.pending_action = pending_actions.IMPORT
|
task.save()
|
|
# Associate the folder url with the project and task
|
combined_id = "{}_{}".format(project_pk, pk)
|
|
datastore = get_current_plugin().get_global_data_store()
|
datastore.set_json(combined_id, {
|
"ddbUrl": ddb_url,
|
"token": ddb.token,
|
"ddbWebUrl": "{}/r/{}/{}/{}".format(to_web_protocol(registry_url), orgSlug, dsSlug, folder.rstrip('/'))
|
})
|
|
#ddb.refresh_token()
|
|
# Start importing the files in the background
|
serialized = {'token': ddb.token, 'files': files}
|
run_function_async(import_files, task.id, serialized)
|
|
return Response({}, status=status.HTTP_200_OK)
|
|
def import_files(task_id, carrier):
|
import requests
|
from app import models
|
from app.plugins import logger
|
from app.security import path_traversal_check
|
|
files = carrier['files']
|
|
headers = {}
|
|
if carrier['token'] != None:
|
headers['Authorization'] = 'Bearer ' + carrier['token']
|
|
def download_file(task, file):
|
path = path_traversal_check(task.task_path(file['name']), task.task_path())
|
logger.info("Downloading file: " + file['url'])
|
download_stream = requests.get(file['url'], stream=True, timeout=60, headers=headers)
|
|
with open(path, 'wb') as fd:
|
for chunk in download_stream.iter_content(4096):
|
fd.write(chunk)
|
|
logger.info("Will import {} files".format(len(files)))
|
task = models.Task.objects.get(pk=task_id)
|
task.create_task_directories()
|
task.save()
|
|
try:
|
downloaded_total = 0
|
for file in files:
|
download_file(task, file)
|
task.check_if_canceled()
|
models.Task.objects.filter(pk=task.id).update(upload_progress=(float(downloaded_total) / float(len(files))))
|
downloaded_total += 1
|
|
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
|
raise NodeServerError(e)
|
|
task.refresh_from_db()
|
task.pending_action = None
|
task.processing_time = 0
|
task.partial = False
|
task.save()
|
|
class CheckUrlTaskView(TaskView):
|
def get(self, request, project_pk=None, pk=None):
|
|
# Assert that task exists
|
self.get_and_check_task(request, pk)
|
|
# Check if there is an imported url associated with the project and task
|
combined_id = "{}_{}".format(project_pk, pk)
|
data = get_current_plugin().get_global_data_store().get_json(combined_id, default = None)
|
|
if data == None or 'ddbWebUrl' not in data:
|
return Response({'ddbWebUrl': None}, status=status.HTTP_200_OK)
|
else:
|
return Response({'ddbUrl': data['ddbWebUrl']}, status=status.HTTP_200_OK)
|
|
def get_status_key(task_id):
|
return '{}_ddb'.format(task_id)
|
|
@receiver(plugin_signals.task_removed, dispatch_uid="ddb_on_task_removed")
|
@receiver(plugin_signals.task_completed, dispatch_uid="ddb_on_task_completed")
|
def ddb_cleanup(sender, task_id, **kwargs):
|
|
from app.plugins import logger
|
|
# When a task is removed, simply remove clutter
|
# When a task is re-processed, make sure we can re-share it if we shared a task previously
|
|
logger.info("Cleaning up DroneDB datastore for task {}".format(str(task_id)))
|
|
datastore = get_current_plugin().get_global_data_store()
|
status_key = get_status_key(task_id)
|
|
logger.info("Info task {0} ({1})".format(str(task_id), status_key))
|
|
datastore.del_key(status_key)
|
|
|
class StatusTaskView(TaskView):
|
def get(self, request, pk):
|
|
task = self.get_and_check_task(request, pk)
|
|
# Associate the folder url with the project and task
|
status_key = get_status_key(pk)
|
|
datastore = get_current_plugin().get_global_data_store()
|
|
task_info = datastore.get_json(status_key, {
|
'status': 0, # Idle
|
'shareUrl': None,
|
'uploadedFiles': 0,
|
'totalFiles': 0,
|
'uploadedSize': 0,
|
'totalSize': 0,
|
'error': None
|
})
|
|
return Response(task_info, status=status.HTTP_200_OK)
|
|
DRONEDB_ASSETS = [
|
'orthophoto.tif',
|
'orthophoto.png',
|
'georeferenced_model.laz',
|
'dtm.tif',
|
'dsm.tif',
|
'shots.geojson',
|
'report.pdf',
|
'ground_control_points.geojson'
|
]
|
|
class ShareTaskView(TaskView):
|
def post(self, request, pk):
|
|
from app.plugins import logger
|
|
task = self.get_and_check_task(request, pk)
|
|
status_key = get_status_key(pk)
|
|
datastore = get_current_plugin().get_global_data_store()
|
|
data = {
|
'status': 1, # Running
|
'shareUrl': None,
|
'uploadedFiles': 0,
|
'totalFiles': 0,
|
'uploadedSize': 0,
|
'totalSize': 0,
|
'error': None
|
}
|
|
datastore.set_json(status_key, data)
|
|
settings = get_settings(request)
|
|
available_assets = [task.get_asset_file_or_stream(f) for f in list(set(task.available_assets) & set(DRONEDB_ASSETS))]
|
|
if 'textured_model.zip' in task.available_assets:
|
texture_files = [join(task.assets_path('odm_texturing'), f) for f in listdir(task.assets_path('odm_texturing')) if isfile(join(task.assets_path('odm_texturing'), f))]
|
available_assets.extend(texture_files)
|
|
assets_path = task.assets_path()
|
|
files = [{'path': f, 'name': f[len(assets_path)+1:], 'size': os.path.getsize(f)} for f in available_assets]
|
|
share_to_ddb.delay(pk, settings, files)
|
|
return Response(data, status=status.HTTP_200_OK)
|
|
|
@task
|
def share_to_ddb(pk, settings, files):
|
|
from app.plugins import logger
|
|
status_key = get_status_key(pk)
|
datastore = get_current_plugin().get_global_data_store()
|
|
registry_url, username, password, token = settings
|
|
ddb = DroneDB(registry_url, username, password, token)
|
|
# Init share (to check)
|
share_token = ddb.share_init()
|
|
status = datastore.get_json(status_key)
|
|
status['totalFiles'] = len(files)
|
status['totalSize'] = sum(i['size'] for i in files)
|
|
datastore.set_json(status_key, status)
|
|
for file in files:
|
|
# check that file exists
|
if not os.path.exists(file['path']):
|
logger.info("File {} does not exist".format(file['path']))
|
continue
|
|
attempt = 0
|
|
while attempt < 3:
|
try:
|
|
attempt += 1
|
|
up = ddb.share_upload(share_token, file['path'], file['name'])
|
|
logger.info("Uploaded " + file['name'] + " to Dronedb (hash: " + up['hash'] + ")")
|
|
status['uploadedFiles'] += 1
|
status['uploadedSize'] += file['size']
|
|
datastore.set_json(status_key, status)
|
|
break
|
|
except Exception as e:
|
|
if (attempt == 3):
|
logger.error("Error uploading file {}: {}".format(file['name'], str(e)))
|
status['error'] = "Error uploading file {}: {}".format(file['name'], str(e))
|
status['status'] = 2 # Error
|
datastore.set_json(status_key, status)
|
return
|
else:
|
logger.info("Error uploading file {}: {}. Retrying...".format(file['name'], str(e)))
|
time.sleep(5)
|
continue
|
|
|
res = ddb.share_commit(share_token)
|
|
status['status'] = 3 # Done
|
status['shareUrl'] = registry_url + res['url']
|
|
logger.info("Shared on url " + status['shareUrl'])
|
|
datastore.set_json(status_key, status)
|
