geonode
geonode copied to clipboard
GeoNode
Implement file upload in GeoNode Django Rest Framework
I am trying to upload a simple textfile as a Geonode document to Geonode via the Django Rest Framework. I know the file upload using GeoNode DRF is not implemented. I also know that the Geonode DRF is based on the DRF on the Github respository.
I want to implement the file upload facility in Geonode DRF. I have been going in circles looking at the Python code - I know I would have to modify the request.py file, but how?
Anyone can give me some pointers as to where to start, or maybe some file I can replace in DRF and it would start to work. I am posting my FETCH CODE which is not working because the file handler code is not implemented.
console.log("Trying to upload document 213345")
//now to send the document upload in geonode
let url = 'http://127.0.0.1:4040/api/v2/documents/';
var crf_token1 = $('[name="csrfmiddlewaretoken"]').attr('value');
//method where we use a token
let headers = new Headers();
var crf_token = $('[name="csrfmiddlewaretoken"]').attr('value');
headers.set('X-CSRFToken', crf_token);
// lets first get a good sample file
var ds_url = 'http://127.0.0.1:4040/static/ds_assets/dtest.txt';
var fileD;
fetch(ds_url)
.then( res => res.blob() )
.then( blob => {
fileD = window.URL.createObjectURL(blob);
});
var formData = new FormData();
//fetch related parameters
formData.append('file', fileD);
formData.append('owner', 'admin'); formData.append('title', 'testdoc');
formData.append('abstract', 'testdoc'); formData.append('attribution', 'test');
formData.append('bbox_polygon', 'MULTIPOLYGON (((0 0, 0 1, 1 1, 1 0, 0 0)))');
formData.append('constraints_other', 'no'); formData.append('data_quality_statement', 'no');
formData.append('doi', 'test'); formData.append('embed_url', 'test');
formData.append('is_approved', 'true'); formData.append('is_published', 'true');
formData.append('ll_bbox_polygon', 'MULTIPOLYGON (((0 0, 0 1, 1 1, 1 0, 0 0)))');
formData.append('maintenance_frequency', '1'); formData.append('metadata_only', 'true');
formData.append('popular_count', '1'); formData.append('share_count', '1');
formData.append('srid', '1'); formData.append('purpose', '1'); formData.append('rating', '1');
formData.append('language', 'English'); formData.append('supplemental_information', 'No information provided');
formData.append('temporal_extent_end', '2022-02-02 12:21'); formData.append('temporal_extent_start', '2022-02-02 12:21');
formData.append('thumbnail_url', 'test'); formData.append('Regions', 'Global');
formData.append('Responsible', 'admin'); formData.append('date', '2022-02-02 12:21');
formData.append('edition', 'test'); formData.append('date_type', 'test');
fetch(url, {method:'POST',
headers: headers ,
body: formData,
credentials: "same-origin",
})
.then(response => response.json())
.then(json => console.log(json));
Try this instead:
var formData = new FormData()
formData.set("csrfmiddlewaretoken", $('[name="csrfmiddlewaretoken"]').attr('value'))
var filename = "test2.txt"
formData.set("title", filename)
var fileStringArray = [ "It works!" ];
var blobAttrs = { type: "text/plain" };
var file = new File(fileStringArray, filename, blobAttrs);
formData.set("doc_file", file)
formData.set("doc_url", "")
formData.set("permissions", '{"users":{"AnonymousUser":["view_resourcebase","download_resourcebase"]},"groups":{}}')
var result = await fetch("/documents/upload", {
"credentials": "include",
"body": formData,
"method": "POST",
"mode": "cors"
});
console.log(`New document uploaded, it's URL is ${result.url}`)
Also, to upload a GeoJSON file as a new layer:
var headers = new Headers();
var csrftoken = $('[name="csrfmiddlewaretoken"]').attr('value');
headers.set('X-CSRFToken', csrftoken);
var formData = new FormData()
formData.set("time", false)
var filename = "sample.geojson"
var geoJSON = { "type": "FeatureCollection",
"features": [
{ "type": "Feature",
"geometry": {"type": "Point", "coordinates": [102.0, 0.5]},
"properties": {"prop0": "value0"}
},
{ "type": "Feature",
"geometry": {
"type": "LineString",
"coordinates": [
[102.0, 0.0], [103.0, 1.0], [104.0, 0.0], [105.0, 1.0]
]
},
"properties": {
"prop0": "value0",
"prop1": 0.0
}
},
{ "type": "Feature",
"geometry": {
"type": "Polygon",
"coordinates": [
[ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0],
[100.0, 1.0], [100.0, 0.0] ]
]
},
"properties": {
"prop0": "value0",
"prop1": {"this": "that"}
}
}
]
}
var fileStringArray = [ JSON.stringify(geoJSON) ];
var blobAttrs = { type: "application/octet-stream" };
var file = new File(fileStringArray, filename, blobAttrs);
formData.set("base_file", file)
formData.set("geojson_file", file)
formData.set("permissions", '{"users":{"AnonymousUser":["view_resourcebase","download_resourcebase"]},"groups":{}}')
formData.set("charset", "UTF-8")
fetch("/upload/", {
"credentials": "include",
"body": formData,
"headers": headers,
"method": "POST",
"mode": "cors"
}).then(result => result.json()).then(data => console.log(data));
Putting it together, this uploads a layer, waits for it to exist, then uploads a new document, linking it to the freshly created layer:
var geoJSON = {
"type": "FeatureCollection",
"features": [{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [102.0, 0.5]
},
"properties": {
"prop0": "value0"
}
},
{
"type": "Feature",
"geometry": {
"type": "LineString",
"coordinates": [
[102.0, 0.0],
[103.0, 1.0],
[104.0, 0.0],
[105.0, 1.0]
]
},
"properties": {
"prop0": "value0",
"prop1": 0.0
}
},
{
"type": "Feature",
"geometry": {
"type": "Polygon",
"coordinates": [
[
[100.0, 0.0],
[101.0, 0.0],
[101.0, 1.0],
[100.0, 1.0],
[100.0, 0.0]
]
]
},
"properties": {
"prop0": "value0",
"prop1": {
"this": "that"
}
}
}
]
}
var csrftoken = $('[name="csrfmiddlewaretoken"]').attr('value');
upload_layer(geoJSON, "sample.geojson")
function upload_layer(geoJSON, filename = "sample.geojson") {
var headers = new Headers();
headers.set('X-CSRFToken', csrftoken);
var formData = new FormData()
formData.set("time", false)
var fileStringArray = [JSON.stringify(geoJSON)];
var blobAttrs = {
type: "application/octet-stream"
};
var file = new File(fileStringArray, filename, blobAttrs);
formData.set("base_file", file)
formData.set("geojson_file", file)
formData.set("permissions", '{"users":{"AnonymousUser":["view_resourcebase","download_resourcebase"]},"groups":{}}')
formData.set("charset", "UTF-8")
fetch("/upload/", {
"credentials": "include",
"body": formData,
"headers": headers,
"method": "POST",
"mode": "cors"
}).then(result => result.json()).then(function(data) {
console.log(data)
fetch(data.redirect_to).then(result => result.json()).then(function(data) {
console.log(data)
// Get layer ID
var name = data.url.split(":").pop()
check_layer(name)
})
})
}
function check_layer(name) {
// Check if layer exists with this name
fetch("/api/v2/layers/?filter{name}=" + name).then(result => result.json()).then(function(result) {
console.log(result)
if (result.layers.length) {
var layer = result.layers[0]
var layer_id = `type:${layer.polymorphic_ctype_id}-id:${layer.pk}`
upload_document(layer_id)
} else {
// It doesn't exist yet, wait 1 second, then check again
setTimeout(function() {
check_layer(name)
}, 1000)
}
})
}
function upload_document(layer_id) {
var formData = new FormData()
formData.set("csrfmiddlewaretoken", csrftoken)
var filename = "test2.txt"
formData.set("title", filename)
formData.set("links", layer_id)
console.log(`Linking to ${layer_id}`)
var fileStringArray = ["It works!"];
var blobAttrs = {
type: "text/plain"
};
var file = new File(fileStringArray, filename, blobAttrs);
formData.set("doc_file", file)
formData.set("doc_url", "")
formData.set("permissions", '{"users":{"AnonymousUser":["view_resourcebase","download_resourcebase"]},"groups":{}}')
var result = fetch("/documents/upload", {
"credentials": "include",
"body": formData,
"method": "POST",
"mode": "cors"
}).then(function(result) {
if (result.status == 200) {
console.log(`New document uploaded, it's URL is ${result.url}`)
} else {
console.error(result)
}
})
}
We should be able to do this in python as well, right? I see there was a discussion here: :point_up: 26. Februar 2021 13:15
which stated it is/was not ready? @afabiani Any news regarding that API endpoint?
You could port my JS to Python? Reimplement the fetch requests with the requests library etc
Alternatively see https://docs.geonode.org/en/master/admin/async/index.html
I have used Selenium to hack my way through the frontend for v3.3.x.
This was written in a getting-shit-done mood 😓
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import os
# must use chromedriver, since others do not interact
# with hidden elements (such as the multi-upload input-field)
PATH_TO_DRIVER = "./lib/chromedriver_v101.exe" # or use a unix driver
# set folder with files to upload
FOLDER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "img")
# fill files to upload
FILES = []
# you need to auth for uploading anything. Do so with:
USER = os.environ.get("GEONODE_USER", "username")
PASS = os.environ.get("GEONODE_PASS", "password")
# the geonode site to upload to
SITE = 'https://your.geonode.domain'
for f in os.listdir(FOLDER):
f_path = os.path.join(FOLDER, f)
FILES.append(f_path)
driver = webdriver.Chrome(executable_path=PATH_TO_DRIVER)
driver.get(SITE + '/account/login/?next=/')
time.sleep(0.1)
driver.find_element(By.ID, value='id_login').send_keys(USER)
driver.find_element(By.ID, value='id_password').send_keys(PASS)
time.sleep(0.1)
submit_button = driver.find_element(By.CSS_SELECTOR, value='html body div#wrap div.container div.row div.col-md-8 '
'form button.btn.btn-primary')
submit_button.click()
time.sleep(0.2)
for f in FILES:
driver.get(SITE + '/documents/upload')
file_input = driver.find_element(By.ID, value='id_doc_file')
file_input.send_keys(f)
time.sleep(0.2)
driver.find_element(By.ID, value='upload-button').click()
time.sleep(1)
# TODO: use upload info to grab document ID and patch metadata via REST API v2
Hello, I am now trying to upload a shapefile instead of geojson. From the Geonode layers page, when the user wants to download the original dataset, we want the shapefile to be downloaded instead of a geojson.
I modified the suggested code to upload a shapefile instead. But the upload just hangs saying its incomplete.
I changed the code after trying to upload shapefile using the standard geonode interface and monitoring the browser developer tools network tab. I also searched online, but some small detail seems to be missing.
//html - I try to upload a shapefile here
var fileInputX = document.getElementById('avatar');
var filename = fileInputX.files[0].name;
console.log('filename: ' + filename);
//Javascript
var headers = new Headers();
headers.set('X-CSRFToken', csrftoken);
var formData = new FormData()
formData.set("time", false)
//$$ formData.set("base_file", file)
formData.set("base_file", fileInputX.files[0])
//$$ formData.set("geojson_file", file)
formData.set("zip_file", fileInputX.files[0]) //file
//this line of code needed to be changed to ensure that the uploaded layer is only visible to logged-in users
//formData.set("permissions", '{"users":{"AnonymousUser":["view_resourcebase","download_resourcebase"]},"groups":{}}')
formData.set("permissions", '{"users":{},"groups":{}}')
formData.set("charset", "UTF-8")
fetch("/upload/", {
"credentials": "include",
"body": formData,
"headers": headers,
"method": "POST",
"mode": "cors"
}).then(result => result.json()).then(function(data) {
console.log(data)
fetch(data.redirect_to).then(result => result.json()).then(function(data) {
console.log(data)
var name = data.url.split(":").pop() // Get layer ID
check_layer(name)
})
})
This is the results. The upload seems to hang after this output in the browser console.
{url: '/upload/?id=204', status: 'incomplete', success: true, id:
204, redirect_to: 'https://WEBSITE/upload/srs?
id=204&force_ajax=true'}
geonodeUploads.js:171 {url: '/upload/?id=204', status:
'incomplete', success: true, id: 204, redirect_to:
'https://WEBSITE/upload/check?id=204'}
I checked the geoserver error.log and there is nothing entered there.
Hi @sharmapn - the result of fetch("/upload/" appears to return something different for a shapefile. The returned redirect_to parameter looks like /upload/srs?id=221&force_ajax=true". The presence of "srs" suggests geonode is doing something related to coordinate reprojection here. You need to make fetch requests to these redirect_urls, until the returned JSON payload has the key status: "finished". Conceptually, you're "following" the redirects specified in redirect_url. This can be implemented with a recursive function like so:
function follow_redirects(payload) {
console.log(payload)
if (payload.redirect_to) {
fetch(payload.redirect_to).then(result => result.json()).then(function(payload) {
follow_redirects(payload)
})
} else {
var name = payload.url.split(":").pop() // Get layer ID
check_layer(name)
}
}
fetch("/upload/", {
"credentials": "include",
"body": formData,
"headers": headers,
"method": "POST",
"mode": "cors"
}).then(result => result.json()).then(function(data) {
follow_redirects(data)
})