import os
|
import time
|
from worker.celery import app as celery
|
import logging
|
import json
|
|
import requests
|
from PIL import Image
|
from django.contrib.auth.models import User
|
from rest_framework import status
|
from rest_framework.test import APIClient
|
from app.plugins.signals import task_completed
|
from app.tests.classes import BootTransactionTestCase
|
from app.models import Project, Task
|
from nodeodm.models import ProcessingNode
|
from nodeodm import status_codes
|
|
import worker
|
from worker.tasks import TestSafeAsyncResult
|
|
from .utils import start_processing_node, clear_test_media_root, catch_signal
|
|
# We need to test in a TransactionTestCase because
|
# task processing happens on a separate thread, and normal TestCases
|
# do not commit changes to the DB, so spawning a new thread will show no
|
# data in it.
|
from webodm import settings
|
logger = logging.getLogger('app.logger')
|
|
DELAY = 2 # time to sleep for during process launch, background processing, etc.
|
|
class TestApiTask(BootTransactionTestCase):
|
def setUp(self):
|
super().setUp()
|
|
def tearDown(self):
|
clear_test_media_root()
|
|
def test_exports(self):
|
client = APIClient()
|
|
with start_processing_node():
|
user = User.objects.get(username="testuser")
|
self.assertFalse(user.is_superuser)
|
|
other_user = User.objects.get(username="testuser2")
|
|
project = Project.objects.create(
|
owner=user,
|
name="test project"
|
)
|
other_project = Project.objects.create(
|
owner=other_user,
|
name="another test project"
|
)
|
|
# Start processing node
|
|
# Create processing node
|
pnode = ProcessingNode.objects.create(hostname="localhost", port=11223)
|
|
# task creation via file upload
|
image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb')
|
image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb')
|
|
client.login(username="testuser", password="test1234")
|
|
# Normal case with images[], name and processing node parameter
|
res = client.post("/api/projects/{}/tasks/".format(project.id), {
|
'images': [image1, image2],
|
'name': 'test task',
|
'processing_node': pnode.id
|
}, format="multipart")
|
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
|
image1.close()
|
image2.close()
|
|
# Should have returned the id of the newly created task
|
task = Task.objects.latest('created_at')
|
|
params = [
|
('orthophoto', {'formula': 'NDVI', 'bands': 'RGN'}, status.HTTP_200_OK),
|
('dsm', {'epsg': 4326}, status.HTTP_200_OK),
|
('dtm', {'epsg': 4326}, status.HTTP_200_OK),
|
('georeferenced_model', {'epsg': 4326}, status.HTTP_200_OK),
|
('georeferenced_model', {'epsg': 4326, 'resample': 2.5}, status.HTTP_200_OK)
|
]
|
|
# Cannot export stuff
|
for p in params:
|
asset_type, data, _ = p
|
res = client.post("/api/projects/{}/tasks/{}/{}/export".format(project.id, task.id, asset_type), data)
|
self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
|
|
# Assign processing node to task via API
|
res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), {
|
'processing_node': pnode.id
|
})
|
self.assertTrue(res.status_code == status.HTTP_200_OK)
|
|
retry_count = 0
|
while task.status != status_codes.COMPLETED:
|
worker.tasks.process_pending_tasks()
|
time.sleep(DELAY)
|
task.refresh_from_db()
|
retry_count += 1
|
if retry_count > 10:
|
break
|
|
self.assertEqual(task.status, status_codes.COMPLETED)
|
|
# Can export stuff (basic)
|
for p in params:
|
asset_type, data, exp_status = p
|
res = client.post("/api/projects/{}/tasks/{}/{}/export".format(project.id, task.id, asset_type), data)
|
self.assertEqual(res.status_code, exp_status)
|
reply = json.loads(res.content.decode("utf-8"))
|
self.assertTrue("celery_task_id" in reply)
|
celery_task_id = reply["celery_task_id"]
|
|
# More exhaustive export testing
|
params = [
|
('orthophoto', {}, True, ".tif", status.HTTP_200_OK),
|
('orthophoto', {'format': 'gtiff'}, True, ".tif", status.HTTP_200_OK),
|
('orthophoto', {'format': 'gtiff-rgb', 'rescale': "10,100"}, False, ".tif", status.HTTP_200_OK),
|
('orthophoto', {'format': 'laz'}, False, ".tif", status.HTTP_400_BAD_REQUEST),
|
('orthophoto', {'format': 'jpg', 'epsg': 4326}, False, ".jpg", status.HTTP_200_OK),
|
('orthophoto', {'format': 'jpg', 'epsg': 4326, 'rescale': '10,200'}, False, ".jpg", status.HTTP_200_OK),
|
('orthophoto', {'format': 'png'}, False, ".png", status.HTTP_200_OK),
|
('orthophoto', {'format': 'kmz'}, False, ".kmz", status.HTTP_200_OK),
|
|
('orthophoto', {'formula': 'NDVI'}, False, "-NDVI.tif", status.HTTP_400_BAD_REQUEST),
|
('orthophoto', {'bands': 'RGN'}, False, "-NDVI.tif", status.HTTP_400_BAD_REQUEST),
|
('orthophoto', {'bands': 'RGN', 'formula': 'NDVI'}, False, "-NDVI.tif", status.HTTP_200_OK),
|
|
('dsm', {'format': 'gtiff'}, True, ".tif", status.HTTP_200_OK),
|
('dsm', {'epsg': 4326}, False, ".tif", status.HTTP_200_OK),
|
('dsm', {'format': 'jpg', 'epsg': 4326}, False, ".jpg", status.HTTP_200_OK),
|
('dsm', {'format': 'jpg', 'color_map': 'jet', 'hillshade': 0, 'epsg': 3857}, False, ".jpg", status.HTTP_200_OK),
|
('dsm', {'epsg': 4326, 'format': 'jpg'}, False, ".jpg", status.HTTP_200_OK),
|
('dsm', {'epsg': 4326, 'format': 'gtiff-rgb'}, False, ".tif", status.HTTP_200_OK),
|
('dsm', {'format': 'kmz'}, False, ".kmz", status.HTTP_200_OK),
|
('dsm', {'color_map': 'viridis', 'hillshade': 2, 'format': 'png'}, False, ".png", status.HTTP_200_OK),
|
('dsm', {'rescale': 'invalid-but-works-cuz-gtiff'}, True, ".tif", status.HTTP_200_OK),
|
|
('dsm', {'epsg': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST),
|
('dsm', {'format': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST),
|
('dsm', {'hillshade': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST),
|
('dsm', {'color_map': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST),
|
('dsm', {'format': 'gtiff-rgb', 'rescale': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST),
|
('dsm', {'format': 'las'}, False, ".tif", status.HTTP_400_BAD_REQUEST),
|
|
('dtm', {'format': 'gtiff'}, True, ".tif", status.HTTP_200_OK),
|
('dtm', {'epsg': 4326}, False, ".tif", status.HTTP_200_OK),
|
|
('georeferenced_model', {}, True, ".laz", status.HTTP_200_OK),
|
('georeferenced_model', {'format': 'las'}, False, ".las", status.HTTP_200_OK),
|
('georeferenced_model', {'format': 'ply'}, False, ".ply", status.HTTP_200_OK),
|
('georeferenced_model', {'format': 'csv'}, False, ".csv", status.HTTP_200_OK),
|
('georeferenced_model', {'format': 'las', 'epsg': 4326}, False, ".las", status.HTTP_200_OK),
|
|
('georeferenced_model', {'format': 'tif'}, False, ".laz", status.HTTP_400_BAD_REQUEST),
|
]
|
|
for p in params:
|
asset_type, data, shortcut_link, extension, exp_status = p
|
logger.info("Testing {}".format(p))
|
res = client.post("/api/projects/{}/tasks/{}/{}/export".format(project.id, task.id, asset_type), data)
|
self.assertEqual(res.status_code, exp_status)
|
|
reply = json.loads(res.content.decode("utf-8"))
|
|
if res.status_code == status.HTTP_200_OK:
|
self.assertTrue("filename" in reply)
|
self.assertEqual(reply["filename"], "test-task-" + asset_type + extension)
|
|
if shortcut_link:
|
self.assertFalse("celery_task_id" in reply)
|
self.assertTrue("url" in reply)
|
|
# Can download
|
res = client.get(reply["url"])
|
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
else:
|
self.assertTrue("celery_task_id" in reply)
|
self.assertFalse("url" in reply)
|
|
cres = TestSafeAsyncResult(celery_task_id)
|
c = 0
|
while not cres.ready():
|
time.sleep(0.2)
|
c += 1
|
if c > 50:
|
self.assertTrue(False)
|
break
|
|
res = client.get("/api/workers/get/{}?filename={}".format(celery_task_id, reply["filename"]))
|
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
self.assertEqual(res._headers['content-disposition'][1], 'attachment; filename={}'.format(reply["filename"]))
|
else:
|
self.assertTrue(len(reply[0]) > 0) # Error message
|
