Commit ed3d6fdd authored by Knut Behrends's avatar Knut Behrends
Browse files

begin work on sample-inserter

parent e409b375
"use strict"
const querystring = require("querystring")
async function upload(ax, url, post_data, i) {
try {
console.log({ [`${i}`]: JSON.stringify(post_data) })
//console.log({ [`${i}`]: JSON.stringify(post_data) })
const response = await ax.post(url, post_data)
} catch (error) {
// console.error(error.config.data)
console.error(error.response.data.name)
console.error(error.response.data.message)
console.error(error.response.statusText)
console.error(error.response?.data?.name)
console.error(error.response?.data?.message)
console.error(error.response?.statusText)
}
}
......@@ -18,9 +18,9 @@ async function update_item(ax, url, put_data, i) {
response = await ax.put(url, put_data)
} catch (error) {
// console.error(error.config.data)
console.error(error.response.data.name)
console.error(error.response.data.message)
console.error(error.response.statusText)
console.error(error.response?.data?.name)
console.error(error.response?.data?.message)
console.error(error.response?.statusText)
}
return new Promise((resolve, reject) => {
resolve(response)
......@@ -33,7 +33,6 @@ async function get_columns(ax, model_name) {
let model = {}
try {
const response = await ax.get(base_url)
if (response?.data?.models?.length > 0) {
response.data.models
.find((m) => m.table.includes(model_name))
......@@ -98,12 +97,7 @@ async function get_item(ax, item_url, n = 0) {
let item = {}
try {
const response = await ax.get(item_url)
if (
response.data !== undefined &&
response.data.items !== undefined &&
Array.isArray(response.data.items) &&
response.data.items.length > 0
) {
if (response.data?.items?.length > 0) {
item = response.data.items[n]
}
} catch (error) {
......@@ -120,16 +114,18 @@ async function get_items(ax, items_url) {
let items = {}
try {
const response = await ax.get(items_url)
if (
response.data !== undefined &&
response.data.items !== undefined &&
Array.isArray(response.data.items) &&
response.data.items.length > 0
) {
if (response.data?.items?.length > 0) {
items = response.data.items // throw away data.meta object
}
} catch (error) {
console.error(error)
// console.dir(error, { depth: 0 })
console.error(items_url.replace(/%5B/g, "[").replace(/%5D/g, "]"))
const name = error.response?.data?.name
if (name) console.error(name)
const msg = error.response?.data?.message
if (msg) console.error(msg)
const txt = error.response?.statusText
if (txt) console.error(txt)
}
// make it chainable again
return new Promise((resolve, reject) => {
......
......@@ -103,4 +103,19 @@ CoreDefaults.sample_material = {
Sediment: "",
Soil: "",
}
CoreDefaults.sample_state = [
"BROKEN",
"CHIPS",
"DECOMP",
"DISC",
"FRAGS",
"GROUND",
"INTACT",
"MIXED",
"MUD",
"SAND",
"WEATH",
]
module.exports = CoreDefaults
......@@ -4,31 +4,226 @@
const faker = require("faker")
const Util = require("./fake-util")
const CoreDefaults = require("./fake-core-defaults")
//faker.seed(112)
const util = new Util()
module.exports = class SampleRequest {
constructor(props, section, prev_section) {
constructor(props, scientists, expedition_id) {
this._props = props
this._section = section
this._prev_section = prev_section
this._scientists = scientists
this._expedition_id = expedition_id
this.samplers = util
.shuffle(this._scientists.map((s) => s.full_name))
.slice(-3)
for (let k of [
"id",
"archive_files",
"combined_id",
"igsn",
"ukbgs_section_id",
"ukbgs_sample_id",
"igsn_ukbgs",
]) {
delete this._props[k]
}
}
find_request_id() {
// redundant. in DB this is a string col but should be int
return this.request_no
}
find_request_no() {
// redundant. in DB this is an int col but should be string
const r = faker.fake(`{{random.number}}`)
this._props.request_no = r
return r
}
find_request_part() {
return faker.fake(`{{name.prefix}}`).slice(-1)
}
find_request_type() {
return util.shuffle(random.number).pop()
}
find_request_type() {
return util.shuffle(CoreDefaults.request_type).pop()
}
find_scientist_1() {
return this.samplers[0]
}
find_scientist_2() {
return Math.random() > util.frac_above(0.75) ? this.samplers[1] : null
}
find_scientist_3() {
// only rarely, and only if scientist_2 exists
const is_n3 = this.scientists_2 && Math.random() > util.frac_above(0.95)
return is_n3 ? this.samplers[2] : null
}
find_purpose() {
//return faker.fake(`${1} {{}} {{}}`)
return util.shuffle(Object.keys(CoreDefaults.sample_purpose)).pop()
//(`${1} {{}} {{}}`)
}
find_request_date() {
return util.format_date(util.date_between(), "YYYY-MM-DD")
}
find_approval_date() {
const d1 = new Date(Date.parse(this.request_date))
const d2 = util.add_offset(d1, Number.parseInt(Math.random() * 50))
return Math.random() > 0.6
? util.format_date(util.date_between(d1, d2), "YYYY-MM-DD")
: null
}
find_completion_date() {
if (!this.approval_date) {
return null
} else {
const d1 = new Date(Date.parse(this.approval_date))
const d2 = util.add_offset(d1, Number.parseInt(Math.random() * 10))
return Math.random() > 0.6
? util.format_date(util.date_between(d1, d2), "YYYY-MM-DD")
: null
}
}
find_remarks() {
return faker.fake(`{{company.bsBuzz}}`)
}
find_sample_material() {
return Math.random() > 0.75
? null
: util.shuffle(CoreDefaults.sample_material).pop()
}
find_sample_volume() {
return util.round(util.frac_above(0.5) * 20)
}
find_request_complete() {
return this.completion_date ? "yes" : "no"
}
find_sample_amount() {
return this.sample_volume ? null : util.round(util.frac_above(0.5) * 20)
}
find_sample_unit() {
return util.shuffle(Object.keys(CoreDefaults.sample_purpose)).pop()
}
find_analyst() {
return util.shuffle(CoreDefaults.curator).pop()
}
find_expedition_id() {
return this._expedition_id
}
fake() {
Object.keys(this._props).forEach((property) => {
this._props[property] = this[property]
})
this._props.section = this._section_num
return this._props
}
get request_id() {
return this._props.request_id
? this._props.request_id
: this.find_request_id()
}
get request_part() {
return this._props.request_part
? this._props.request_part
: this.find_request_part()
}
get expedition_id() {
return this._props.expedition_id
? this._props.expedition_id
: this.find_expedition_id()
}
get request_id() {
return this._props.request_id
? this._props.request_id
: this.find_request_id()
}
get request_type() {
return this._props.request_types
? this._props.request_type
: this.find_request_type()
}
get scientist_1() {
return this._props.scientist_1
? this._props.scientist_1
: this.find_scientist_1()
}
get scientist_2() {
return this._props.scientist_2
? this._props.scientist_2
: this.find_scientist_2()
}
get scientist_3() {
return this._props.scientist_3
? this._props.scientist_3
: this.find_scientist_3()
}
get purpose() {
return this._props.purpose ? this._props.purpose : this.find_purpose()
}
get request_date() {
return this._props.request_date
? this._props.request_date
: this.find_request_date()
}
get approval_date() {
return this._props.approval_date
? this._props.approval_date
: this.find_approval_date()
}
get completion_date() {
return this._props.completion_date
? this._props.completion_date
: this.find_completion_date()
}
get remarks() {
return this._props.remarks ? this._props.remarks : this.find_remarks()
}
get sample_material() {
return this._props.sample_material
? this._props.sample_material
: this.find_sample_material()
}
get sample_volume() {
return this._props.sample_volume
? this._props.sample_volume
: this.find_sample_volume()
}
get request_no() {
return this._props.request_no
? this._props.request_no
: this.find_request_no()
}
get request_complete() {
return this._props.request_complete
? this._props.request_complete
: this.find_request_complete()
}
get sample_amount() {
return this._props.sample_amount
? this._props.sample_amount
: this.find_sample_amount()
}
get sample_unit() {
return this._props.sample_unit
? this._props.sample_unit
: this.find_sample_unit()
}
get analyst() {
return this._props.analyst ? this._props.analyst : this.find_analyst()
}
get program_id() {
return this._props.program_id
? this._props.program_id
: this.find_program_id()
}
}
......@@ -63,19 +63,7 @@ module.exports = class CoreSectionSplit {
find_comment() {
let cond = util
.shuffle([
"BROKEN",
"CHIPS",
"DECOMP",
"DISC",
"FRAGS",
"GROUND",
"INTACT",
"MIXED",
"MUD",
"SAND",
"WEATH",
])
.shuffle(CoreDefaults.sample_state)
.map((c) => c.toLowerCase())
.slice(-2)
.join(", ")
......
#!/usr/bin/env node
"use strict"
// knb July 2020
// use Node v14+
// insert sample-requests into an mDIS database
//
// call this script with:
// ../api-caller-request-inserter.sh
// or
// node src/request-inserter.js
const config = require("./config.js")
const axios = require("axios")
const axutil = require("./axios-util.js")
const querystring = require("querystring")
const getopts = require("getopts")
const Util = require("./classes/fake-util.js")
const SampleRequest = require("./classes/fake-sample-requests.js")
const util = new Util()
const options = getopts(process.argv.slice(2), {
alias: {
expedition_id: "e", // PK column 'id' of core_core table
//infile: "i", // alternative
},
default: {
expedition_id: 10, //CAIR
},
})
//let core_url = "/api/v1/form?name=core&per-page=1&sort=-id"
//const core_info = options.infile ? JSON.parse(options.infile) : {}
const ax = axios.create(config.endpoint)
const api_url_frag = `/api/v1/form`
// al sections of
const qs_scientists = {
name: "scientists",
"per-page": -1,
page: 1,
sort: "city",
"filter[expedition_id]": options.expedition_id,
}
const qs_requests = {
name: "sample-requests",
"per-page": -1,
page: 1,
sort: "id",
"filter[expedition_id]": options.expedition_id,
}
let scientists = {}
const scientist_get_url =
api_url_frag + "?" + querystring.stringify(qs_scientists)
let requests = {}
const requests_get_url = api_url_frag + "?" + querystring.stringify(qs_requests)
console.log(querystring.unescape(scientist_get_url))
console.log(querystring.unescape(requests_get_url))
// get section for this core from section table
axutil
.get_items(ax, requests_get_url)
.then((data) => (requests = data))
.then((_) => {
if (!requests.length) {
console.log(
`Attention!
`
)
}
console.log(
` Expedition ${options.expedition_id} has 0 sample requests so far.`
)
})
.then(() => {
// only include cores that have whole rounds WR
// and that do not have A and W items (s.exists)
axutil
.get_items(ax, scientist_get_url)
.then((data) => {
scientists = data
if (!scientists.length) {
console.log(
`Attention!
`
)
console.log(
` Expedition ${options.expedition_id} has ${scientists.length} 0 scientists registered.
Nothing to do. Create some fake scientists first, or associate them with different Expeditions`
)
process.exit(0)
} else {
console.log(
` Expedition ${options.expedition_id} has ${scientists.length} scientists registered.`
)
}
return new Promise((resolve, reject) => {
resolve(scientists)
reject(scientists)
})
})
.then((scientists) => {
// create template object-literals for fake splits
axutil
.get_columns(ax, "sample_request")
.then((request_columns) => {
let fake_request_data = []
let i = 0
for (let s of scientists) {
const n_requests = 15
const n_max = 10
// create an array of 0-10 requests per scientist
let requests_per_scientists = Array(n_requests)
.fill(scientists.length, 1)
.map((_) => Number.parseInt(util.frac_below(0.2) * n_max))
for (let r of requests_per_scientists.filter((n) => n > 0)) {
let templ = new SampleRequest(
Object.assign({}, request_columns, { request_id: "CAIR" }),
scientists,
options.expedition_id
).fake()
fake_request_data.push(
Object.assign({}, templ, {
"expedition_id": options.expedition_id,
})
)
i++
}
}
// console.log(fake_request_data[0])
return new Promise((resolve, reject) => {
resolve(fake_request_data), reject(fake_request_data)
})
})
.then((fake_request_data) => {
return fake_request_data.flat().map((fake_request, i) => {
axutil.upload(
ax,
`${api_url_frag}?name=sample-requests`,
fake_request,
i
)
})
})
.then((fake_request_promises) =>
Promise.all(fake_request_promises).then((s) =>
console.log(`
#############################################################
Expedition ${options.expedition_id}: Tried to create ${fake_request_promises.length} sample-requests
############################################################# `)
)
)
.catch((error) => console.dir(error, { depth: 0 }))
// .then((_) => {
// // set 'sampling' attribute from '-' to 'Y', or similar
// for (let s of suitable_sections) {
// let url = `${api_url_frag}/${s.id}?name=split`
// let put_data = {
// sampleable: "",
// remarks: `core ${options.expdition_id}`,
// }
// axutil.update_item(ax, url, put_data, s.id).then((updated) => {
// console.log(
// {
// "'sampleable' is now:": `${updated?.data?.sampleable}`,
// },
// { [`id_${updated?.data?.combined_id}`]: updated?.data?.id }
// )
// })
// }
// })
})
})
.catch((error) => console.dir(error, { depth: 0 }))
......@@ -2,35 +2,28 @@
"use strict"
// knb July 2020
// use Node v14+
// insert samples into an mDIS database
// either - for a single split (can be of type W, WR, but not A)
// insert sample-requests into an mDIS database
//
// call this script with:
// ../api-caller-sample-inserter.sh
// ../api-caller-request-inserter.sh
// or
// node src/sample-inserter.js -c 123
// or
// for i in $(seq 101 200) ; do echo $i; node split-inserter.js -c $i ; done
// node src/request-inserter.js
const config = require("./config.js")
const axios = require("axios")
const axutil = require("./axios-util.js")
const querystring = require("querystring")
const getopts = require("getopts")
const Util = require("./classes/fake-util.js")
const CurationSample = require("./classes/fake-samples.js")
// const SampleRequest = require("./classes/fake-sample-requests.js")
const util = new Util()
const options = getopts(process.argv.slice(2), {
alias: {
split_id: "s", //split
core_id: "c", // PK column 'id' of core_core table
expedition: "e", // PK column 'id' of core_core table
//infile: "i", // alternative
},
default: {
//infile: "sections.json",
// core_id: 95, //ENTY
// split_id: 4136,
core_id: 96,
expedition: 10, //CAIR
},
})
......@@ -39,185 +32,225 @@ const options = getopts(process.argv.slice(2), {
const ax = axios.create(config.endpoint)
const api_url_frag = `/api/v1/form`
// al sections of
const qs_section_split = {
name: "split",
const qs_scientists = {
name: "scientists",
"per-page": -1,
page: 1,
sort: "city",
"filter[expedition_id]": options.expedition,
}
const qs_request = {
name: "sample-requests",
"per-page": -1,
page: 1,
sort: "id",
"filter[expedition_id]": options.expedition,
}
const qs_cores = {
name: "core",
"per-page": -1,
page: 1,
sort: "section",
"filter[core_id]": options.core_id,
sort: "id",
"filter[expedition_id]": options.expedition,
}
const qs_expedition = {
name: "expedition",
"filter[id]": options.expedition,
}
</