Skip to content
GitLab
Commit 49fae30e authored by Knut Behrends's avatar Knut Behrends
Browse files

almost finish inserter for fake-samples :-)

parent a393f32d
Hide whitespace changes
Inline Side-by-side
USAGE.md
View file @ 49fae30e
......@@ -89,4 +89,4 @@ node list-values.js -n CITIES -e
### Configuration
To use the scripts you must get a Bearer token for the REST API first. Add it either as an environment variable, or insert it at the top of the `src/config` file.
To use the scripts you must get a [Bearer token](https://data.icdp-online.org/mdis-docs/guide/rest-api/#security) for the REST API first. Add it either as an environment variable, or insert it at the top of the `src/config` file.
src/axios-util.js
View file @ 49fae30e
......@@ -2,7 +2,7 @@
const querystring = require("querystring")
async function upload(ax, url, post_data, i) {
try {
//console.log({ [`${i}`]: JSON.stringify(post_data) })
console.log({ [`${i}`]: JSON.stringify(post_data) })
const response = await ax.post(url, post_data)
} catch (error) {
// console.error(error.config.data)
......@@ -35,8 +35,9 @@ async function get_columns(ax, model_name) {
const response = await ax.get(base_url)
if (response.data?.models?.length > 0) {
model = response.data.models.find((m) => m.table.includes(model_name))
model.columns.forEach((_) => (model[_] = undefined))
response.data.models
.find((m) => m.table.includes(model_name))
.columns.forEach((_) => (model[_] = undefined))
}
} catch (error) {
console.error(error.response?.data?.name)
......
src/classes/fake-samples.js
View file @ 49fae30e
......@@ -3,14 +3,15 @@
//var moment = require("moment")
const faker = require("faker")
const CoreDefaults = require("./fake-core-defaults.js")
const Util = require("./fake-util")
//faker.seed(112)
const util = new Util()
module.exports = class CurationSample {
constructor(props, section) {
constructor(props, section_split) {
this._props = props
this._section = section
this._section = section_split
for (let k of [
"id",
"archive_files",
......@@ -27,36 +28,58 @@ module.exports = class CurationSample {
return this._props.section_split_id
}
find_sample_date() {
return this._props.section_split_id
return this._props.sample_date
}
find_top() {
return (
this._section.top_depth +
util.frac_below(0.3) * this._section.section_length
)
find_bottom() {
const b = util.round((
this.top + this.interval
), 1)
this._props.bottom = b
return b
}
find_top() {
return (
const t = util.round((
this._section.top_depth +
util.frac_below(0.3) * this._section.section_length
)
util.frac_below(0.3) * (this._section.section_length * 100) //cm
), 1)
this._props.top = t
return t
}
find_interval() {
return Math.random()
const sl = this._section.section_length * 100
const minlen = 0.03 * (sl + 1)
let len = 0
if(sl < 10){
// allow larger pieces for smaller sections
len = sl * util.frac_above(0.3)
} else {
len = sl * util.frac_below(0.2)
}
const i = util.round((minlen + len) ) * 100
this._props.interval= i
return i
}
find_analyst() {
return this._section.curator
return util.shuffle(CoreDefaults.curator).pop()
}
find_sample_type() {
return util.shuffle(CoreDefaults.sample_type).pop()
return util.shuffle(CoreDefaults.sample_purpose).pop()
//return this._props.sample_type
}
find_section_top_mbsf() {
return faker.fake("{{random.number}}") % 200
const t = this._section.top_depth + 1000 //faker.fake("{{random.number}}") % 200
this._props.section_top_mbsf = t
return t
}
find_request_id() {
return this._props.request_id
}
find_request_no() {
return this._section.request_no
return this.request_id
}
find_comment() {
......@@ -66,10 +89,10 @@ module.exports = class CurationSample {
.slice(-2)
.join(", ")
return faker.fake(`core ${this._section.core_id} `)
return `core id: ${this._section.core_id} ${this._props.remarks}`
}
find_amount() {
return faker.fake("{{random.float}}")
return util.round((faker.fake("{{random.number}}") % 200) * util.frac_above(0.5), 0)
}
find_amount_unit() {
return util.shuffle(CoreDefaults.sampling_units).pop()
......@@ -81,24 +104,25 @@ module.exports = class CurationSample {
return this._section.sample_volume
}
find_sample_top_mbsf() {
return faker.fake("{{random.number}}") % 300
let t = util.round(this.section_top_mbsf + this.top, 1)
this._props.sample_top_mbsf = t
return t
}
find_sample_bottom_mbsf() {
return this.sample_top_mbsf + (faker.fake("{{random.number}}") % 300)
return util.round(this.section_top_mbsf + this.top + this.interval, 1)
}
find_scientist() {
return Array.of(
this._props.scientist_1,
this._props.scientist_2,
this._props.scientist_3
).filter((s) => typeof s != "undefined")
).filter((s) => s?.length ).join(", ")
}
fake() {
Object.keys(this._props).forEach((property) => {
this._props[property] = this[property]
})
this._props.section = this._section_num
return this._props
}
......
src/sample-inserter.js
View file @ 49fae30e
......@@ -14,7 +14,7 @@ const axutil = require("./axios-util.js")
const querystring = require("querystring")
const getopts = require("getopts")
const Util = require("./classes/fake-util.js")
// const SampleRequest = require("./classes/fake-sample-requests.js")
const CurationSample = require("./classes/fake-samples.js")
const util = new Util()
const options = getopts(process.argv.slice(2), {
......@@ -69,6 +69,9 @@ const requests_get_url = api_url_frag + "?" + querystring.stringify(qs_request)
const splits_get_url = api_url_frag + "?" + querystring.stringify(qs_splits)
const samples_get_url = api_url_frag + "?" + querystring.stringify(qs_samples)
delete qs_samples["filter[expedition_id]"]
const samples_post_url = api_url_frag + "?" + querystring.stringify(qs_samples)
// let qs = [qs_splits, qs_request, qs_scientists, qs_expedition]
// // check if cmdline arg was numerc or acronym
// Number.parseInt(options.expedition)
......@@ -154,7 +157,7 @@ Promise.all(queries_map.values())
(Total Requests: ${req_len_before} for Expedition ${options.expedition}${exp_name}.)
########################################################################`)
let section_split_id_set = new Set(splits.map((s) => s.section_id))
let section_split_id_set = new Set(splits.map((s) => s.id))
scientists.forEach((s) => (s.city = s.city.toUpperCase()))
scientists.sort(str_sort_by_city).map((sc) => {
......@@ -169,24 +172,26 @@ Promise.all(queries_map.values())
)
scientists_by_city.set(sc.city, {
city: sc.city,
split: spl,
split_section: spl,
requests: req,
})
})
// make sure all teams from all cities sample at least one section
for (let [k, c] of scientists_by_city.entries()) {
if (!c.split.length) {
if (!c.split_section.length) {
let randsection = util
.shuffle(Array.from(section_split_id_set.values()))
.pop()
// This step might pick "untouchable" "A" section-splits for sampling,
// but that's okay. Correct manually later
let sections_split_ids = Array.of(
randsection - Math.trunc(Math.random() * 10),
randsection,
randsection + Math.trunc(Math.random() * 10)
)
c.split.push(
c.split_section.push(
splits.find((sp) =>
sections_split_ids.some((ssi) => sp.section_id === ssi)
sections_split_ids.some((ssi) => sp.id === ssi)
)
)
}
......@@ -195,8 +200,8 @@ Promise.all(queries_map.values())
for (let c of scientists_by_city.values()) {
console.log({
"city": c.city,
"spli": c.split.map((v) => ({
[v.section_id]: v.combined_id,
"spli": c.split_section.map((v) => ({
[v.id]: v.combined_id,
})),
"req": Array(c.requests.map((v) => v.request_id)).join(", "),
})
......@@ -204,19 +209,67 @@ Promise.all(queries_map.values())
})
.then(() => {
let curation_sample_columns = {}
axutil.get_columns(ax, "curation_sample_2").then((obj) => {
curation_sample_columns = obj
console.log(curation_sample_columns)
// for (let c of scientists_by_city.values()) {
// for (let req in c.requests) {
// r
// }
// }
axutil.get_columns(ax, "curation_sample_2")
.then((curation_sample) => {
let fake_sample_data = []
let i = 0
for (let s of scientists_by_city.values()) {
// create an array of 0-10 requests per scientist,
// assign a few echo sections_split(s) "reserved" for this team
// (usually there are only 1-2 splits for this team)
for (let sp of s.split_section){
let n_picked = Number.parseInt(util.frac_below(0.666) * s.requests.length)
let request_ids_per_city = util.shuffle(s.requests).slice(-n_picked)
// ~ 5 requests for this split
for (let r of request_ids_per_city) {
let fake_sample = new CurationSample(
Object.assign({}, curation_sample, {
request_id: r.request_id,
request_no: `${r.request_no}`,
scientist_1: r.scientist_1,
scientist_2: r.scientist_2,
scientist_3: r.scientist_3,
section_split_id: sp.id,
remarks: `SecId:${sp.section_id}, secLen ${sp.section_length}`,
sample_date : Array.of(r.completion_date, r.approval_date,
r.request_date, util.date_between()).find(d => d != null)
}),
sp
).fake()
fake_sample_data.push(
Object.assign({}, fake_sample)
)
i++
}
}
}
console.log(`Created ${i} fake samples, will try to upload`)
// console.log(fake_sample_data[0])
return new Promise((resolve, reject) => {
resolve(fake_sample_data), reject(fake_sample_data)
})
})
.then((fake_sample_data) => {
return fake_sample_data.flat().map((fake_sample, i) => {
axutil.upload(
ax,
samples_post_url,
fake_sample,
i
)
})
})
.then((fake_request_promises) =>
Promise.all(fake_request_promises).then((s) =>
console.log(`
#############################################################
Expedition ${options.expedition}: Tried to upload ${fake_request_promises.length} samples
############################################################# `)
)
)
})
})
})
.catch((error) => console.dir(error, { depth: 0 }))
.catch((error) => console.dir(error, { depth: 0 }))
function str_sort_by_city(a, b) {
let A = a.city.toUpperCase()
......@@ -241,53 +294,6 @@ function str_sort_by_site_hole(a, b) {
}
return 0
}
//console.log({ type: typeof _, keys: Object.keys(_).pop() })
// axutil
// .get_items(ax, scientist_get_url)
// .then((scientists) => {
// scientists = data
// // later, for each scientists, find how many requests there are
// let scientists_map = new Map()
// axutil
// .get_items(ax, requests_get_url)
// .then((data) => (requests = data))
// .then((_) => {
// if (!requests.length) {
// console.log(
// `Attention!
// `
// )
// }
// console.log(
// ` Expedition ${options.expedition} has 0 sample requests so far.`
// )
// })
// .then(() => {
// // only include splits that have whole rounds WR
// // and that do not have A and W items (s.exists)
// axutil
// .get_items(ax, scientist_get_url)
// .then((data) => {
// scientists = data
// if (!scientists.length) {
// console.log(
// `Attention!
// `
// )
// console.log(
// ` Expedition ${options.expedition} has ${scientists.length} 0 scientists registered.
// Nothing to do. Create some fake scientists first, or associate them with different Expeditions`
// )
// process.exit(0)
// } else {
// console.log(
// ` Expedition ${options.expedition} has ${scientists.length} scientists registered.`
// )
// }
// return new Promise((resolve, reject) => {
// resolve(scientists)
......@@ -298,55 +304,9 @@ function str_sort_by_site_hole(a, b) {
// // create template object-literals for fake splits
// axutil
// .get_columns(ax, "sample_request")
// .then((request_columns) => {
// let fake_request_data = []
// let i = 0
// for (let s of scientists) {
// const n_requests = 15
// const n_max = 10
// // create an array of 0-10 requests per scientist
// let requests_per_scientists = Array(n_requests)
// .fill(scientists.length, 1)
// .map((_) => Number.parseInt(util.frac_below(0.2) * n_max))
// for (let r of requests_per_scientists.filter((n) => n > 0)) {
// let templ = new SampleRequest(
// Object.assign({}, request_columns, {
// request_id: "CAIR",
// }),
// scientists,
// options.expedition
// ).fake()
// fake_request_data.push(
// Object.assign({}, templ, {
// "expedition_id": options.expedition,
// })
// )
// i++
// }
// }
// // console.log(fake_request_data[0])
// return new Promise((resolve, reject) => {
// resolve(fake_request_data), reject(fake_request_data)
// })
// })
// .then((fake_request_data) => {
// return fake_request_data.flat().map((fake_request, i) => {
// axutil.upload(
// ax,
// `${api_url_frag}?name=sample-requests`,
// fake_request,
// i
// )
// })
// })
// .then((fake_request_promises) =>
// Promise.all(fake_request_promises).then((s) =>
// console.log(`
// #############################################################
// Expedition ${options.expedition}: Tried to create ${fake_request_promises.length} sample-requests
// ############################################################# `)
// )
// )
// .
// .catch((error) => console.dir(error, { depth: 0 }))
// // .then((_) => {
// // // set 'sampling' attribute from '-' to 'Y', or similar
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment