Skip to content
GitLab
Commit dfd29996 authored by Knut Behrends's avatar Knut Behrends
Browse files

90% finish split-inserter, does not issue PUT requests yet

parent de86bf96
Hide whitespace changes
Inline Side-by-side
src/axios-util.js
View file @ dfd29996
......@@ -2,7 +2,7 @@
async function upload(ax, url, post_data, i) {
try {
console.log(i, post_data)
console.log({ [`${i}`]: JSON.stringify(post_data) })
const response = await ax.post(url, post_data)
} catch (error) {
// console.error(error.config.data)
......@@ -100,7 +100,7 @@ async function get_item(ax, item_url, n = 0) {
item = response.data.items[n]
}
} catch (error) {
console.error(error)
console.dir(error, { depth: 0 })
}
// make it chainable again
return new Promise((resolve, reject) => {
......
src/classes/fake-splits.js
View file @ dfd29996
......@@ -4,7 +4,6 @@
const faker = require("faker")
const Util = require("./fake-util")
const CoreDefaults = require("./fake-split-defaults.js")
/* example minimum payload
......@@ -31,12 +30,10 @@ const CoreDefaults = require("./fake-split-defaults.js")
const util = new Util()
module.exports = class CoreSectionSplit {
constructor(props, section, core, prev_section) {
constructor(props, section, prev_section) {
this._props = props
this._section = section
this._core = core
this._prev_section = prev_section
this._section_num = section_stub.section
for (let k of [
"id",
"archive_files",
......@@ -50,77 +47,87 @@ module.exports = class CoreSectionSplit {
}
find_percent() {
return 50
return Math.random() > 0.05 ? 50 : util.round(util.frac_below(0.5) * 100, 0)
}
find_section_id() {
return util.shuffle([1, 2, 3, 4])
return util.shuffle([1, 2, 3, 4]).pop()
}
find_type() {
return faker.fake(util.shuffle(["A", "W"]))
return util.shuffle(["A", "W"]).pop()
}
find_origin_split_id() {
return faker.fake("{{}}{{}}")
return this._section.id
}
find_exists() {
return util.frac_above(0.05) ? 1 : 0
return Math.random() > 0.05 ? 1 : 0
}
find_sampleable() {
return util.frac_above(0.05) ? 1 : 0
return Math.random() > 0.05 ? 1 : 0
}
find_box() {
return util.shuffle(["BW", "BA"])
return util.shuffle([1, 2, 3, 4]).pop()
}
find_slot() {
return util.shuffle([1, 2, 3, 4])
return util.shuffle([1, 2, 3, 4]).pop()
}
find_position() {
return util.shuffle(["T", "M", "M", "M", "M", "B", "F"])
return util.shuffle(["T", "M", "M", "M", "M", "B", "F"]).pop()
}
find_repository() {
return util.shuffle([
"AWI",
"BCR",
"BGR",
"BGS",
"CGS",
"GCR",
"KCR",
"LACCORE",
"ONSITE",
"USGS",
])
return util
.shuffle([
"AWI",
"BCR",
"BGR",
"BGS",
"CGS",
"GCR",
"KCR",
"LACCORE",
"ONSITE",
"USGS",
])
.pop()
}
find_shelf() {
return [
"AWI",
"BCR",
"BGR",
"BGS",
"CGS",
"GCR",
"KCR",
"LACCORE",
"ONSITE",
"USGS",
]
return util.shuffle("ABCDEFGHI".split("")).pop()
}
find_compartment() {
return faker.fake(["top", "middle", "bottom"])
return util.shuffle(["top", "middle", "bottom"]).pop()
}
find_remarks() {
let cond = util
.shuffle([
"BROKEN",
"CHIPS",
"DECOMP",
"DISC",
"FRAGS",
"GROUND",
"INTACT",
"MIXED",
"MUD",
"SAND",
"WEATH",
])
.map((c) => c.toLowerCase())
.slice(-2)
.join(", ")
return faker.fake(
"{{company.catchPhraseAdjective}} {{name.firstName name.LastName}}"
`{{company.catchPhraseAdjective}}, ${cond} ({{name.firstName}} {{name.lastName}})`
)
}
find_curated_length() {
return Math.random > 0.95
? this.section_length
: frac_above(0.95) * this.section_length
: util.round(util.frac_above(0.95) * this.section_length)
}
find_curator() {
return util.shuffle(CoreDefaults.curator)
return Math.random() > 0.95
? util.shuffle(["CK", "KB", "KH", "TK", "UR"]).pop()
: this.curator
}
find_top_depth() {
return this._section.top_depth
......@@ -129,27 +136,22 @@ module.exports = class CoreSectionSplit {
return this._section.bottom_depth
}
find_section_length() {
return this._section.curated_length
return this._section.section_length
}
find_origin_split_type() {
return "BW"
}
find_test() {
return faker.fake("{{lorem.word}}")
return undefined //faker.fake("{{lorem.word}}")
}
find_core_id() {
return this._core.id)
return this._section.core_id
}
fake() {
Object.keys(this._props).forEach((property) => {
this._props[property] = this[property]
})
// let d =
// this._section_num !== 1 && this._prev_section
// ? this._prev_section.bottom_depth + this.section_length
// : this._core.bottom_depth + this.section_length
// this._props.bottom_depth = util.round(d)
this._props.section = this._section_num
return this._props
}
......
src/config.js
View file @ dfd29996
let token = process.env.token || "mb99LO9qn5a5uAQDzkHUtfpbjLZ24wfy"
let token = process.env.token || "BTEJmYlstdpCpVes7GecvPTm6AcAfvdh"
let config = {
token: token,
endpoint: {
......
src/split-inserter.js
View file @ dfd29996
......@@ -25,6 +25,7 @@ const axutil = require("./axios-util.js")
const querystring = require("querystring")
const getopts = require("getopts")
const Util = require("./classes/fake-util.js")
const CoreSectionSplit = require("./classes/fake-splits.js")
const util = new Util()
const options = getopts(process.argv.slice(2), {
......@@ -42,135 +43,141 @@ const options = getopts(process.argv.slice(2), {
//let core_url = "/api/v1/form?name=core&per-page=1&sort=-id"
//const core_info = options.infile ? JSON.parse(options.infile) : {}
const ax = axios.create(config.endpoint)
const report_url_frag = "/report/SplitSection"
const api_url_frag = `/api/v1/form?`
const qs_core_section = {
// al sections of
const qs_section_split = {
name: "split",
"per-page": -1,
page: 1,
sort: "section",
"filter[core_id]": options.core_id,
}
const qs_section_split = {
name: "split",
"per-page": -1,
page: 1,
sort: "section_id",
//"filter[section_id]": undefined,
}
const qs_report = { "model": "CoreSection", "id": undefined }
let core_sections = {}
const section_get_url = api_url_frag + querystring.stringify(qs_core_section)
let split_report_url = report_url_frag + querystring.stringify(qs_report)
let sections_map = new Map()
let split_sections_by_core = {}
const split_section_get_url =
api_url_frag + querystring.stringify(qs_section_split)
let split_sections_by_core_map = new Map()
console.log(querystring.unescape(section_get_url))
console.log(querystring.unescape(split_section_get_url))
// get section for this core from section table
axutil
.get_items(ax, section_get_url)
.then((data) => (core_sections = data))
.get_items(ax, split_section_get_url)
.then((data) => (split_sections_by_core = data))
.then((_) => {
const rx = /(_A|_WR|_W)$/gi
// sort sections by combined_id
// let sections_map = new Map(
// [...core_sections.entries()].sort(
// (a, b) => a[1].combined_id - b[1].combined_id
// )
if (!core_sections.length) {
if (!split_sections_by_core.length) {
console.log(
`Done. Core ${core_sections[0].combined_id_core} (id ${options.core_id})has 0 ${sections_map.size} sections.
`Stopped early. Nothing to do. Core id ${options.core_id} has 0 sections.
Create some fake sections for this core first.`
)
process.exit(0)
}
})
.then((_) => {
// get all sections for this core.
// All splits, whole - round and archive / working
// set up a structure finding what was already split
let types = {}
// core_sections.forEach((_) => (types[_.type] = 0))
core_sections.forEach((cs) =>
let rx = new RegExp("_A$|_W$|_WR$", "i")
split_sections_by_core.forEach((cs) =>
["A", "W", "WR"].forEach((t) => (types[t] = 0))
)
core_sections.forEach((_) => {
// initialize an object of all split-types,
// set count for all split - types to 0
split_sections_by_core.forEach((_) => {
const combined_id_core = _.combined_id.replace(rx, "")
_.combined_id_core = combined_id_core
sections_map.set(combined_id_core, Object.assign({}, types))
split_sections_by_core_map.set(combined_id_core, Object.assign({}, types))
})
//
Array(sections_map.keys()).forEach((_) => {
core_sections.forEach((_) => {
let tmp = sections_map.get(_.combined_id_core)
})
.then(() => {
// set count=1 for the respective split-types found
split_sections_by_core_map.forEach((v, k, map) => {
split_sections_by_core.forEach((_) => {
let tmp = split_sections_by_core_map.get(_.combined_id_core)
tmp[`${_.type}`] = 1
sections_map.set(_.combined_id_core, tmp)
split_sections_by_core_map.set(_.combined_id_core, tmp)
})
})
//core_sections = util.map2object(sections_map)
//console.table(sections_map)
let splittable_sections_ids = []
})
.then(() => {
// only include cores that have whole rounds WR
// and that do not have A and W items
sections_map.forEach((v, k, map) => {
// and that do not have A and W items (s.exists)
let splittable_sections_ids = []
split_sections_by_core_map.forEach((v, k, map) => {
if (v.A === 0 && v.W === 0 && v.WR === 1) {
splittable_sections_ids.push(k)
}
})
// console.log(sections_map)
// console.log(splittable_sections_ids)
if (!splittable_sections_ids.length) {
console.log(
`Done. Core ${core_sections[0].combined_id_core} (id ${options.core_id}) and its ${sections_map.size} sections
seems to be split already, at least partially.`
`Stop early. Core ${split_sections_by_core[0].combined_id_core} (id ${options.core_id}) and its ${split_sections_by_core_map.size} sections
seems to be split already, at least partially.
Maybe create the missing split sections manually.`
)
return 0
process.exit(0)
} else {
console.log(
` Core ${core_sections[0].combined_id_core} (id ${options.core_id}) has ${splittable_sections_ids.length} / ${sections_map.size} splittable sections.`
` Core ${split_sections_by_core[0].combined_id_core} (id ${options.core_id}) has ${splittable_sections_ids.length} / ${split_sections_by_core_map.size} splittable sections.`
)
}
let splittable_sections = core_sections.filter((s) =>
return new Promise((resolve, reject) => {
resolve(splittable_sections_ids.sort())
reject(splittable_sections_ids)
})
})
.then((splittable_sections_ids) => {
// filter sections again for appropriate WR sections
// now also apply "still_exists" filter
let splittable_sections = split_sections_by_core.filter((s) =>
splittable_sections_ids.some(
(v) => s.combined_id_core === v && s.type === "WR"
(v) => s.combined_id_core === v && s.type === "WR" && s.exists === 1
)
)
//console.log(JSON.stringify(splittable_sections[0]))
//console.log(splittable_sections)
//console.log(core_sections[0])
let fake_splits = []
let core_section_split_columns = {}
axutil.get_columns(ax, "split").then((obj) => {
core_section_split_columns = obj
console.log(core_section_split_columns)
return new Promise((resolve, reject) => {
resolve(splittable_sections)
reject(splittable_sections)
})
/*
.then((_) => {
for (const v of splittable_sections) {
let prev_section =
v.section > 1 && fake_splits[v.section - 2]
? fake_splits[v.section - 2]
: {}
let cp = {}
cp = Object.assign(
cp,
new CoreSectionSplit(
core_section_split_columns,
v,
core,
prev_section
})
.then((splittable_sections) => {
// create template object-literals for fake splits
axutil
.get_columns(ax, "split")
.then((core_section_split_columns) => {
let fake_section_data = []
for (let s of splittable_sections.sort()) {
let i = 0
for (let t of ["A", "W"]) {
let templ = new CoreSectionSplit(
Object.assign({}, core_section_split_columns),
Object.assign({}, s),
i === 0 ? {} : s[i - 1]
).fake()
)
fake_splits.push(cp)
fake_section_data.push(
Object.assign({}, templ, {
"percent": 50,
"section_id": s.section_id,
"type": t,
"origin_split_id": "WR",
"origin_split_type": "WR",
"test": null,
"core_id": options.core_id,
})
)
}
i += 1
}
return new Promise((resolve, reject) => {
resolve(fake_section_data), reject(fake_section_data)
})
})
.then((_) => {
axutil
.insert_split(ax, section_url, core_info, i)
.then((s) => sections.set(s.section, s))
.catch(function (error) {
console.error(error)
})
.then((fake_section_data) => {
return fake_section_data.flat().map((fake_split, i) => {
axutil.upload(ax, `${api_url_frag}name=split`, fake_split, i)
})
})
*/
//axutil.post()
.then((fake_split_promises) =>
Promise.all(fake_split_promises).then((s) => console.log("Done"))
)
// TO DO issue a put request that split no longer exists
})
.catch((error) => console.dir(error, { depth: 0 }))
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment