DataDepsGenerators.jl
DataDepsGenerators.jl copied to clipboard
oxinabox
Support projects at osf.io
I have done some exploring of the API at https://api.osf.io. The way that I would see generating a DataDep for files in a project on that site is
- start with a project id, say
kz483which is the project "Stable Preview Difficulty Effects"
julia> JSON.print(JSON.parse(String(HTTP.get("https://api.osf.io/v2/nodes/kz483").body)),2)
{
"meta": {
"version": "2.0"
},
"data": {
"links": {
"self": "https://api.osf.io/v2/nodes/kz483/",
"html": "https://osf.io/kz483/"
},
"attributes": {
"public": true,
"date_created": "2018-08-20T14:52:20.727535",
"preprint": false,
"registration": false,
"current_user_permissions": [
"read"
],
"description": "Parafoveal word processing can affect the reader's eyes in at least two different ways. The present study focuses on the spatio-temporal effects of the preview difficulty and discusses implications for models of eye movement control during reading. ",
"analytics_key": "84a49b0b67bd5a658a7f874d3f28007002adeacbb9f63236f7a6e15aa270af164fb050af6e220de246b51048ee444a0564bdd00582ac9c9f46d0d5a571865eed14a9a9d4e1df39b7ee3be6d6b949c5aeab72a9650f34dd1529047a535b4c17e81e230faccd4bdea149d04f017043fe02fa02196a11b1ad8659e8ecc3f6e0d1b6c42088ff47fbfe10ba160cceafc33354",
"date_modified": "2018-08-21T11:21:42.324633",
"fork": false,
"subjects": [],
"category": "project",
"current_user_is_contributor": false,
"current_user_can_comment": false,
"node_license": null,
"access_requests_enabled": true,
"title": "Stable Preview Difficulty Effects",
"tags": [
"boundary paradigm",
"delayed parafoveal-on-foveal effect",
"eye movements",
"E-Z Reader",
"forced fixations",
"gaze-contingent display changes",
"Glenmore",
"parafoveal-on-foveal effect",
"parafoveal preview benefit",
"reading",
"SWIFT"
],
"custom_citation": null,
"wiki_enabled": true,
"collection": false
},
"id": "kz483",
"type": "nodes",
"relationships": {
"view_only_links": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/view_only_links/"
}
}
},
"node_links": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/node_links/"
}
}
},
"draft_registrations": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/draft_registrations/"
}
}
},
"affiliated_institutions": {
"links": {
"self": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/relationships/institutions/"
},
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/institutions/"
}
}
},
"root": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
},
"data": {
"id": "kz483",
"type": "nodes"
}
},
"forks": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/forks/"
}
}
},
"linked_by_nodes": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/linked_by_nodes/"
}
}
},
"preprints": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/preprints/"
}
}
},
"wikis": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/wikis/"
}
}
},
"logs": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/logs/"
}
}
},
"linked_by_registrations": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/linked_by_registrations/"
}
}
},
"settings": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/settings/"
}
}
},
"files": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/files/"
}
}
},
"registrations": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/registrations/"
}
}
},
"comments": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/comments/?filter%5Btarget%5D=kz483"
}
}
},
"children": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/children/"
}
}
},
"identifiers": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/identifiers/"
}
}
},
"linked_registrations": {
"links": {
"self": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/relationships/linked_registrations/"
},
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/linked_registrations/"
}
}
},
"citation": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/citation/"
}
}
},
"region": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/regions/us/"
}
},
"data": {
"id": "us",
"type": "regions"
}
},
"contributors": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/contributors/"
}
}
},
"linked_nodes": {
"links": {
"self": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/relationships/linked_nodes/"
},
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/linked_nodes/"
}
}
},
"implicit_contributors": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/implicit_contributors/"
}
}
}
}
}
}
The files relationship takes us to osfstorage
julia> JSON.print(JSON.parse(String(HTTP.get("https://api.osf.io/v2/nodes/kz483/files").body)),2)
{
"meta": {
"version": "2.0"
},
"links": {
"prev": null,
"meta": {
"total": 1,
"per_page": 10
},
"next": null,
"first": null,
"last": null
},
"data": [
{
"links": {
"new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/?kind=folder",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/",
"storage_addons": "https://api.osf.io/v2/addons/?filter%5Bcategories%5D=storage"
},
"attributes": {
"kind": "folder",
"name": "osfstorage",
"provider": "osfstorage",
"node": "kz483",
"path": "/"
},
"id": "kz483:osfstorage",
"type": "files",
"relationships": {
"files": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/"
}
}
}
}
}
]
}
within which there is information on several file folders, including R_Data.
julia> JSON.print(JSON.parse(String(HTTP.get("https://api.osf.io/v2/nodes/kz483/files/osfstorage").body)),2)
{
"meta": {
"version": "2.0"
},
"links": {
"prev": null,
"meta": {
"total": 4,
"per_page": 10
},
"next": null,
"first": null,
"last": null
},
"data": [
{
"links": {
"delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adf0684f087001cdb4431/",
"self": "https://api.osf.io/v2/files/5b7adf0684f087001cdb4431/",
"new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adf0684f087001cdb4431/?kind=folder",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adf0684f087001cdb4431/",
"info": "https://api.osf.io/v2/files/5b7adf0684f087001cdb4431/",
"move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adf0684f087001cdb4431/"
},
"attributes": {
"date_created": null,
"current_version": 1,
"materialized_path": "/R_Analyses/",
"path": "/5b7adf0684f087001cdb4431/",
"name": "R_Analyses",
"guid": null,
"date_modified": null,
"extra": {
"hashes": {
"sha256": null,
"md5": null
}
},
"last_touched": null,
"checkout": null,
"size": null,
"current_user_can_comment": false,
"kind": "folder",
"provider": "osfstorage",
"tags": []
},
"id": "5b7adf0684f087001cdb4431",
"type": "files",
"relationships": {
"files": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/5b7adf0684f087001cdb4431/"
}
}
},
"target": {
"links": {
"related": {
"meta": {
"type": "node"
},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
}
},
"node": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
},
"data": {
"id": "kz483",
"type": "nodes"
}
}
}
},
{
"links": {
"delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfa925e63c0018e407e2/",
"self": "https://api.osf.io/v2/files/5b7adfa925e63c0018e407e2/",
"new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfa925e63c0018e407e2/?kind=folder",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfa925e63c0018e407e2/",
"info": "https://api.osf.io/v2/files/5b7adfa925e63c0018e407e2/",
"move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfa925e63c0018e407e2/"
},
"attributes": {
"date_created": null,
"current_version": 1,
"materialized_path": "/R_Functions/",
"path": "/5b7adfa925e63c0018e407e2/",
"name": "R_Functions",
"guid": null,
"date_modified": null,
"extra": {
"hashes": {
"sha256": null,
"md5": null
}
},
"last_touched": null,
"checkout": null,
"size": null,
"current_user_can_comment": false,
"kind": "folder",
"provider": "osfstorage",
"tags": []
},
"id": "5b7adfa925e63c0018e407e2",
"type": "files",
"relationships": {
"files": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/5b7adfa925e63c0018e407e2/"
}
}
},
"target": {
"links": {
"related": {
"meta": {
"type": "node"
},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
}
},
"node": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
},
"data": {
"id": "kz483",
"type": "nodes"
}
}
}
},
{
"links": {
"delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/",
"self": "https://api.osf.io/v2/files/5b7adfc325e63c0019e40676/",
"new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/?kind=folder",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/",
"info": "https://api.osf.io/v2/files/5b7adfc325e63c0019e40676/",
"move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/"
},
"attributes": {
"date_created": null,
"current_version": 1,
"materialized_path": "/R_Data/",
"path": "/5b7adfc325e63c0019e40676/",
"name": "R_Data",
"guid": null,
"date_modified": null,
"extra": {
"hashes": {
"sha256": null,
"md5": null
}
},
"last_touched": null,
"checkout": null,
"size": null,
"current_user_can_comment": false,
"kind": "folder",
"provider": "osfstorage",
"tags": []
},
"id": "5b7adfc325e63c0019e40676",
"type": "files",
"relationships": {
"files": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/5b7adfc325e63c0019e40676/"
}
}
},
"target": {
"links": {
"related": {
"meta": {
"type": "node"
},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
}
},
"node": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
},
"data": {
"id": "kz483",
"type": "nodes"
}
}
}
},
{
"links": {
"delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7ae01425e63c0018e40863/",
"self": "https://api.osf.io/v2/files/5b7ae01425e63c0018e40863/",
"new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7ae01425e63c0018e40863/?kind=folder",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7ae01425e63c0018e40863/",
"info": "https://api.osf.io/v2/files/5b7ae01425e63c0018e40863/",
"move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7ae01425e63c0018e40863/"
},
"attributes": {
"date_created": null,
"current_version": 1,
"materialized_path": "/Manuscript/",
"path": "/5b7ae01425e63c0018e40863/",
"name": "Manuscript",
"guid": null,
"date_modified": null,
"extra": {
"hashes": {
"sha256": null,
"md5": null
}
},
"last_touched": null,
"checkout": null,
"size": null,
"current_user_can_comment": false,
"kind": "folder",
"provider": "osfstorage",
"tags": []
},
"id": "5b7ae01425e63c0018e40863",
"type": "files",
"relationships": {
"files": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/5b7ae01425e63c0018e40863/"
}
}
},
"target": {
"links": {
"related": {
"meta": {
"type": "node"
},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
}
},
"node": {
"links": {
"related": {
"meta": {},
"href": "https://api.osf.io/v2/nodes/kz483/"
}
},
"data": {
"id": "kz483",
"type": "nodes"
}
}
}
}
]
}
and, within that directory, the files to download
julia> JSON.print(JSON.parse(String(HTTP.get("https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/").body)),2)
{
"data": [
{
"links": {
"delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfd384f087001cdb4514",
"download": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfd384f087001cdb4514",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfd384f087001cdb4514?kind=file",
"move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfd384f087001cdb4514"
},
"attributes": {
"path": "/5b7adfd384f087001cdb4514",
"resource": "kz483",
"name": "pb2_n0.rda",
"contentType": "application/octet-stream",
"etag": "837bdf186ba88e402c0d273a2d9f5c62d3bad67b44bbc97dc4d3501eab01d78a",
"extra": {
"downloads": 4,
"latestVersionSeen": null,
"checkout": null,
"hashes": {
"sha256": "1ceb4c1defde581b0b4cf89725fb665b618d0fab47c900875731480632e60284",
"md5": "1efd49f796f39427c53719cbc45dd425"
},
"guid": null,
"version": 1
},
"modified_utc": "2018-08-20T15:35:47.815296+00:00",
"created_utc": "2018-08-20T15:35:47.815296+00:00",
"size": 417405,
"provider": "osfstorage",
"kind": "file",
"modified": "2018-08-20T15:35:47.815296+00:00",
"sizeInt": 417405,
"materialized": "/R_Data/pb2_n0.rda"
},
"id": "osfstorage/5b7adfd384f087001cdb4514",
"type": "files"
},
{
"links": {
"delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfdb84f087001bdb429d",
"download": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfdb84f087001bdb429d",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfdb84f087001bdb429d?kind=file",
"move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfdb84f087001bdb429d"
},
"attributes": {
"path": "/5b7adfdb84f087001bdb429d",
"resource": "kz483",
"name": "pb2_n1.rda",
"contentType": "application/octet-stream",
"etag": "1158afb0ab29e41be2eaef74a2d63a8a02fd55d5b1828017dc20461223bed9db",
"extra": {
"downloads": 3,
"latestVersionSeen": null,
"checkout": null,
"hashes": {
"sha256": "ba12a76ded3720182864505095e607e62228ea4d9ed2afedcd3b4771c13e33be",
"md5": "a58ac2e0a9092feea70f14590146d711"
},
"guid": null,
"version": 1
},
"modified_utc": "2018-08-20T15:35:55.705473+00:00",
"created_utc": "2018-08-20T15:35:55.705473+00:00",
"size": 359380,
"provider": "osfstorage",
"kind": "file",
"modified": "2018-08-20T15:35:55.705473+00:00",
"sizeInt": 359380,
"materialized": "/R_Data/pb2_n1.rda"
},
"id": "osfstorage/5b7adfdb84f087001bdb429d",
"type": "files"
},
{
"links": {
"delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe025e63c0019e406a1",
"download": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe025e63c0019e406a1",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe025e63c0019e406a1?kind=file",
"move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe025e63c0019e406a1"
},
"attributes": {
"path": "/5b7adfe025e63c0019e406a1",
"resource": "kz483",
"name": "pb2_n2.rda",
"contentType": "application/octet-stream",
"etag": "baba1dbf08c5a0b6c90d233419c8116e88ff998e7136b8feef2ab6b4d2257db8",
"extra": {
"downloads": 2,
"latestVersionSeen": null,
"checkout": null,
"hashes": {
"sha256": "d2de410cfebed370bf4f525512f847991c7ed2a7bc20a71a43aa36f6e689d571",
"md5": "513c3603111f2ff821ff7f32de53393d"
},
"guid": null,
"version": 1
},
"modified_utc": "2018-08-20T15:36:00.347061+00:00",
"created_utc": "2018-08-20T15:36:00.347061+00:00",
"size": 326401,
"provider": "osfstorage",
"kind": "file",
"modified": "2018-08-20T15:36:00.347061+00:00",
"sizeInt": 326401,
"materialized": "/R_Data/pb2_n2.rda"
},
"id": "osfstorage/5b7adfe025e63c0019e406a1",
"type": "files"
},
{
"links": {
"delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe484f087001cdb4531",
"download": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe484f087001cdb4531",
"upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe484f087001cdb4531?kind=file",
"move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe484f087001cdb4531"
},
"attributes": {
"path": "/5b7adfe484f087001cdb4531",
"resource": "kz483",
"name": "typeNgramfreqs.RData",
"contentType": "application/octet-stream",
"etag": "a99e130396dd632feb2fd8fc9084336add922bb57b38f0d9bf722112d38c7261",
"extra": {
"downloads": 6,
"latestVersionSeen": null,
"checkout": null,
"hashes": {
"sha256": "f5511bc475d89358f1bcf7cd15d01aee7d5e228b28477c5079115d57a4e9f695",
"md5": "659538398510833d7c461a49143a0a45"
},
"guid": null,
"version": 1
},
"modified_utc": "2018-08-20T15:36:04.354320+00:00",
"created_utc": "2018-08-20T15:36:04.354320+00:00",
"size": 11646,
"provider": "osfstorage",
"kind": "file",
"modified": "2018-08-20T15:36:04.35432+00:00",
"sizeInt": 11646,
"materialized": "/R_Data/typeNgramfreqs.RData"
},
"id": "osfstorage/5b7adfe484f087001cdb4531",
"type": "files"
}
]
}
The actual download link has a hexadecimal name but the information on what it should be named as a file is available in the attributes.
@oxinabox Can you tell me if you think this would fit in with the DataDepsGenerators scheme and, if so, where I should start?
Yes, this look like it should work well.
Take a look at https://github.com/oxinabox/DataDepsGenerators.jl/blob/master/src/APIs/Figshare.jl
Or really anything else from the APIs folder. There are basically a set of methods to overload. To add support for a new API generator.
You can see how all the methods get invoked Here https://github.com/oxinabox/DataDepsGenerators.jl/blob/master/src/generate.jl#L71
