begin work on sample-inserter

src/axios-util.js

 "use strict"
-
+const querystring = require("querystring")
 async function upload(ax, url, post_data, i) {
   try {
-    console.log({ [`${i}`]: JSON.stringify(post_data) })
+    //console.log({ [`${i}`]: JSON.stringify(post_data) })
     const response = await ax.post(url, post_data)
   } catch (error) {
     // console.error(error.config.data)
-    console.error(error.response.data.name)
-    console.error(error.response.data.message)
-    console.error(error.response.statusText)
+    console.error(error.response?.data?.name)
+    console.error(error.response?.data?.message)
+    console.error(error.response?.statusText)
   }
 }
 
@@ -18,9 +18,9 @@ async function update_item(ax, url, put_data, i) {
     response = await ax.put(url, put_data)
   } catch (error) {
     // console.error(error.config.data)
-    console.error(error.response.data.name)
-    console.error(error.response.data.message)
-    console.error(error.response.statusText)
+    console.error(error.response?.data?.name)
+    console.error(error.response?.data?.message)
+    console.error(error.response?.statusText)
   }
   return new Promise((resolve, reject) => {
     resolve(response)
@@ -33,7 +33,6 @@ async function get_columns(ax, model_name) {
   let model = {}
   try {
     const response = await ax.get(base_url)
-
     if (response?.data?.models?.length > 0) {
       response.data.models
         .find((m) => m.table.includes(model_name))
@@ -98,12 +97,7 @@ async function get_item(ax, item_url, n = 0) {
   let item = {}
   try {
     const response = await ax.get(item_url)
-    if (
-      response.data !== undefined &&
-      response.data.items !== undefined &&
-      Array.isArray(response.data.items) &&
-      response.data.items.length > 0
-    ) {
+    if (response.data?.items?.length > 0) {
       item = response.data.items[n]
     }
   } catch (error) {
@@ -120,16 +114,18 @@ async function get_items(ax, items_url) {
   let items = {}
   try {
     const response = await ax.get(items_url)
-    if (
-      response.data !== undefined &&
-      response.data.items !== undefined &&
-      Array.isArray(response.data.items) &&
-      response.data.items.length > 0
-    ) {
+    if (response.data?.items?.length > 0) {
       items = response.data.items // throw away data.meta object
     }
   } catch (error) {
-    console.error(error)
+    // console.dir(error, { depth: 0 })
+    console.error(items_url.replace(/%5B/g, "[").replace(/%5D/g, "]"))
+    const name = error.response?.data?.name
+    if (name) console.error(name)
+    const msg = error.response?.data?.message
+    if (msg) console.error(msg)
+    const txt = error.response?.statusText
+    if (txt) console.error(txt)
   }
   // make it chainable again
   return new Promise((resolve, reject) => {

src/classes/fake-core-defaults.js

@@ -103,4 +103,19 @@ CoreDefaults.sample_material = {
   Sediment: "",
   Soil: "",
 }
+
+CoreDefaults.sample_state = [
+  "BROKEN",
+  "CHIPS",
+  "DECOMP",
+  "DISC",
+  "FRAGS",
+  "GROUND",
+  "INTACT",
+  "MIXED",
+  "MUD",
+  "SAND",
+  "WEATH",
+]
+
 module.exports = CoreDefaults

src/classes/fake-sample-requests.js

@@ -4,31 +4,226 @@
 
 const faker = require("faker")
 const Util = require("./fake-util")
+const CoreDefaults = require("./fake-core-defaults")
 //faker.seed(112)
 const util = new Util()
 
 module.exports = class SampleRequest {
-  constructor(props, section, prev_section) {
+  constructor(props, scientists, expedition_id) {
     this._props = props
-    this._section = section
-    this._prev_section = prev_section
+    this._scientists = scientists
+    this._expedition_id = expedition_id
+    this.samplers = util
+      .shuffle(this._scientists.map((s) => s.full_name))
+      .slice(-3)
+
     for (let k of [
       "id",
       "archive_files",
       "combined_id",
       "igsn",
-      "ukbgs_section_id",
+      "ukbgs_sample_id",
       "igsn_ukbgs",
     ]) {
       delete this._props[k]
     }
   }
 
+  find_request_id() {
+    // redundant. in DB this is a string col but should be int
+    return this.request_no
+  }
+  find_request_no() {
+    // redundant. in DB this is an int col but should be string
+
+    const r = faker.fake(`{{random.number}}`)
+    this._props.request_no = r
+    return r
+  }
+
+  find_request_part() {
+    return faker.fake(`{{name.prefix}}`).slice(-1)
+  }
+
+  find_request_type() {
+    return util.shuffle(random.number).pop()
+  }
+
+  find_request_type() {
+    return util.shuffle(CoreDefaults.request_type).pop()
+  }
+
+  find_scientist_1() {
+    return this.samplers[0]
+  }
+  find_scientist_2() {
+    return Math.random() > util.frac_above(0.75) ? this.samplers[1] : null
+  }
+  find_scientist_3() {
+    // only rarely, and only if scientist_2 exists
+    const is_n3 = this.scientists_2 && Math.random() > util.frac_above(0.95)
+    return is_n3 ? this.samplers[2] : null
+  }
+  find_purpose() {
+    //return faker.fake(`${1} {{}} {{}}`)
+    return util.shuffle(Object.keys(CoreDefaults.sample_purpose)).pop()
+    //(`${1} {{}} {{}}`)
+  }
+  find_request_date() {
+    return util.format_date(util.date_between(), "YYYY-MM-DD")
+  }
+  find_approval_date() {
+    const d1 = new Date(Date.parse(this.request_date))
+    const d2 = util.add_offset(d1, Number.parseInt(Math.random() * 50))
+    return Math.random() > 0.6
+      ? util.format_date(util.date_between(d1, d2), "YYYY-MM-DD")
+      : null
+  }
+  find_completion_date() {
+    if (!this.approval_date) {
+      return null
+    } else {
+      const d1 = new Date(Date.parse(this.approval_date))
+      const d2 = util.add_offset(d1, Number.parseInt(Math.random() * 10))
+      return Math.random() > 0.6
+        ? util.format_date(util.date_between(d1, d2), "YYYY-MM-DD")
+        : null
+    }
+  }
+
+  find_remarks() {
+    return faker.fake(`{{company.bsBuzz}}`)
+  }
+  find_sample_material() {
+    return Math.random() > 0.75
+      ? null
+      : util.shuffle(CoreDefaults.sample_material).pop()
+  }
+  find_sample_volume() {
+    return util.round(util.frac_above(0.5) * 20)
+  }
+
+  find_request_complete() {
+    return this.completion_date ? "yes" : "no"
+  }
+  find_sample_amount() {
+    return this.sample_volume ? null : util.round(util.frac_above(0.5) * 20)
+  }
+  find_sample_unit() {
+    return util.shuffle(Object.keys(CoreDefaults.sample_purpose)).pop()
+  }
+  find_analyst() {
+    return util.shuffle(CoreDefaults.curator).pop()
+  }
+  find_expedition_id() {
+    return this._expedition_id
+  }
+
   fake() {
     Object.keys(this._props).forEach((property) => {
       this._props[property] = this[property]
     })
-    this._props.section = this._section_num
     return this._props
   }
+
+  get request_id() {
+    return this._props.request_id
+      ? this._props.request_id
+      : this.find_request_id()
+  }
+
+  get request_part() {
+    return this._props.request_part
+      ? this._props.request_part
+      : this.find_request_part()
+  }
+  get expedition_id() {
+    return this._props.expedition_id
+      ? this._props.expedition_id
+      : this.find_expedition_id()
+  }
+  get request_id() {
+    return this._props.request_id
+      ? this._props.request_id
+      : this.find_request_id()
+  }
+  get request_type() {
+    return this._props.request_types
+      ? this._props.request_type
+      : this.find_request_type()
+  }
+  get scientist_1() {
+    return this._props.scientist_1
+      ? this._props.scientist_1
+      : this.find_scientist_1()
+  }
+  get scientist_2() {
+    return this._props.scientist_2
+      ? this._props.scientist_2
+      : this.find_scientist_2()
+  }
+  get scientist_3() {
+    return this._props.scientist_3
+      ? this._props.scientist_3
+      : this.find_scientist_3()
+  }
+  get purpose() {
+    return this._props.purpose ? this._props.purpose : this.find_purpose()
+  }
+  get request_date() {
+    return this._props.request_date
+      ? this._props.request_date
+      : this.find_request_date()
+  }
+  get approval_date() {
+    return this._props.approval_date
+      ? this._props.approval_date
+      : this.find_approval_date()
+  }
+  get completion_date() {
+    return this._props.completion_date
+      ? this._props.completion_date
+      : this.find_completion_date()
+  }
+  get remarks() {
+    return this._props.remarks ? this._props.remarks : this.find_remarks()
+  }
+  get sample_material() {
+    return this._props.sample_material
+      ? this._props.sample_material
+      : this.find_sample_material()
+  }
+  get sample_volume() {
+    return this._props.sample_volume
+      ? this._props.sample_volume
+      : this.find_sample_volume()
+  }
+  get request_no() {
+    return this._props.request_no
+      ? this._props.request_no
+      : this.find_request_no()
+  }
+  get request_complete() {
+    return this._props.request_complete
+      ? this._props.request_complete
+      : this.find_request_complete()
+  }
+  get sample_amount() {
+    return this._props.sample_amount
+      ? this._props.sample_amount
+      : this.find_sample_amount()
+  }
+  get sample_unit() {
+    return this._props.sample_unit
+      ? this._props.sample_unit
+      : this.find_sample_unit()
+  }
+  get analyst() {
+    return this._props.analyst ? this._props.analyst : this.find_analyst()
+  }
+  get program_id() {
+    return this._props.program_id
+      ? this._props.program_id
+      : this.find_program_id()
+  }
 }

src/classes/fake-samples.js

@@ -63,19 +63,7 @@ module.exports = class CoreSectionSplit {
 
   find_comment() {
     let cond = util
-      .shuffle([
-        "BROKEN",
-        "CHIPS",
-        "DECOMP",
-        "DISC",
-        "FRAGS",
-        "GROUND",
-        "INTACT",
-        "MIXED",
-        "MUD",
-        "SAND",
-        "WEATH",
-      ])
+      .shuffle(CoreDefaults.sample_state)
       .map((c) => c.toLowerCase())
       .slice(-2)
       .join(", ")

src/request-inserter.js

+#!/usr/bin/env node
+"use strict"
+// knb July 2020
+// use Node v14+
+// insert sample-requests into an mDIS database
+//
+// call this script with:
+// ../api-caller-request-inserter.sh
+// or
+// node src/request-inserter.js
+const config = require("./config.js")
+const axios = require("axios")
+const axutil = require("./axios-util.js")
+const querystring = require("querystring")
+const getopts = require("getopts")
+const Util = require("./classes/fake-util.js")
+const SampleRequest = require("./classes/fake-sample-requests.js")
+const util = new Util()
+
+const options = getopts(process.argv.slice(2), {
+  alias: {
+    expedition_id: "e", // PK column 'id' of core_core table
+    //infile: "i", // alternative
+  },
+  default: {
+    expedition_id: 10, //CAIR
+  },
+})
+
+//let core_url = "/api/v1/form?name=core&per-page=1&sort=-id"
+//const core_info = options.infile ? JSON.parse(options.infile) : {}
+const ax = axios.create(config.endpoint)
+const api_url_frag = `/api/v1/form`
+// al sections of
+const qs_scientists = {
+  name: "scientists",
+  "per-page": -1,
+  page: 1,
+  sort: "city",
+  "filter[expedition_id]": options.expedition_id,
+}
+const qs_requests = {
+  name: "sample-requests",
+  "per-page": -1,
+  page: 1,
+  sort: "id",
+  "filter[expedition_id]": options.expedition_id,
+}
+
+let scientists = {}
+const scientist_get_url =
+  api_url_frag + "?" + querystring.stringify(qs_scientists)
+
+let requests = {}
+const requests_get_url = api_url_frag + "?" + querystring.stringify(qs_requests)
+
+console.log(querystring.unescape(scientist_get_url))
+console.log(querystring.unescape(requests_get_url))
+
+// get section for this core from section table
+axutil
+  .get_items(ax, requests_get_url)
+  .then((data) => (requests = data))
+  .then((_) => {
+    if (!requests.length) {
+      console.log(
+        `Attention!
+        `
+      )
+    }
+    console.log(
+      ` Expedition ${options.expedition_id} has 0 sample requests so far.`
+    )
+  })
+  .then(() => {
+    // only include cores that have whole rounds WR
+    // and that do not have A and W items (s.exists)
+    axutil
+      .get_items(ax, scientist_get_url)
+      .then((data) => {
+        scientists = data
+
+        if (!scientists.length) {
+          console.log(
+            `Attention!
+        `
+          )
+          console.log(
+            ` Expedition ${options.expedition_id} has ${scientists.length} 0 scientists registered.
+              Nothing to do. Create some fake scientists first, or associate them with different Expeditions`
+          )
+          process.exit(0)
+        } else {
+          console.log(
+            ` Expedition ${options.expedition_id} has ${scientists.length} scientists registered.`
+          )
+        }
+
+        return new Promise((resolve, reject) => {
+          resolve(scientists)
+          reject(scientists)
+        })
+      })
+      .then((scientists) => {
+        // create template object-literals for fake splits
+        axutil
+          .get_columns(ax, "sample_request")
+          .then((request_columns) => {
+            let fake_request_data = []
+            let i = 0
+            for (let s of scientists) {
+              const n_requests = 15
+              const n_max = 10
+              // create an array of 0-10 requests per scientist
+              let requests_per_scientists = Array(n_requests)
+                .fill(scientists.length, 1)
+                .map((_) => Number.parseInt(util.frac_below(0.2) * n_max))
+              for (let r of requests_per_scientists.filter((n) => n > 0)) {
+                let templ = new SampleRequest(
+                  Object.assign({}, request_columns, { request_id: "CAIR" }),
+                  scientists,
+                  options.expedition_id
+                ).fake()
+                fake_request_data.push(
+                  Object.assign({}, templ, {
+                    "expedition_id": options.expedition_id,
+                  })
+                )
+                i++
+              }
+            }
+            // console.log(fake_request_data[0])
+            return new Promise((resolve, reject) => {
+              resolve(fake_request_data), reject(fake_request_data)
+            })
+          })
+          .then((fake_request_data) => {
+            return fake_request_data.flat().map((fake_request, i) => {
+              axutil.upload(
+                ax,
+                `${api_url_frag}?name=sample-requests`,
+                fake_request,
+                i
+              )
+            })
+          })
+          .then((fake_request_promises) =>
+            Promise.all(fake_request_promises).then((s) =>
+              console.log(`
+############################################################# 
+Expedition ${options.expedition_id}:  Tried to create ${fake_request_promises.length}  sample-requests 
+############################################################# `)
+            )
+          )
+          .catch((error) => console.dir(error, { depth: 0 }))
+        // .then((_) => {
+        //   // set 'sampling' attribute from '-' to 'Y', or similar
+        //   for (let s of suitable_sections) {
+        //     let url = `${api_url_frag}/${s.id}?name=split`
+        //     let put_data = {
+        //       sampleable: "",
+        //       remarks: `core ${options.expdition_id}`,
+        //     }
+        //     axutil.update_item(ax, url, put_data, s.id).then((updated) => {
+        //       console.log(
+        //         {
+        //           "'sampleable' is now:": `${updated?.data?.sampleable}`,
+        //         },
+        //         { [`id_${updated?.data?.combined_id}`]: updated?.data?.id }
+        //       )
+        //     })
+        //   }
+        // })
+      })
+  })
+  .catch((error) => console.dir(error, { depth: 0 }))

src/sample-inserter.js

@@ -2,35 +2,28 @@
 "use strict"
 // knb July 2020
 // use Node v14+
-// insert samples into an mDIS database
-// either - for a single split (can be of type W, WR, but not A)
+// insert sample-requests into an mDIS database
 //
 // call this script with:
-// ../api-caller-sample-inserter.sh
+// ../api-caller-request-inserter.sh
 // or
-// node src/sample-inserter.js -c 123
-// or
-// for i in $(seq 101 200)  ; do echo $i; node split-inserter.js -c $i ; done
+// node src/request-inserter.js
 const config = require("./config.js")
 const axios = require("axios")
 const axutil = require("./axios-util.js")
 const querystring = require("querystring")
 const getopts = require("getopts")
 const Util = require("./classes/fake-util.js")
-const CurationSample = require("./classes/fake-samples.js")
+// const SampleRequest = require("./classes/fake-sample-requests.js")
 const util = new Util()
 
 const options = getopts(process.argv.slice(2), {
   alias: {
-    split_id: "s", //split
-    core_id: "c", // PK column 'id' of core_core table
+    expedition: "e", // PK column 'id' of core_core table
     //infile: "i", // alternative
   },
   default: {
-    //infile: "sections.json",
-    // core_id: 95, //ENTY
-    // split_id: 4136,
-    core_id: 96,
+    expedition: 10, //CAIR
   },
 })
 
@@ -39,185 +32,225 @@ const options = getopts(process.argv.slice(2), {
 const ax = axios.create(config.endpoint)
 const api_url_frag = `/api/v1/form`
 // al sections of
-const qs_section_split = {
-  name: "split",
+const qs_scientists = {
+  name: "scientists",
+  "per-page": -1,
+  page: 1,
+  sort: "city",
+  "filter[expedition_id]": options.expedition,
+}
+const qs_request = {
+  name: "sample-requests",
+  "per-page": -1,
+  page: 1,
+  sort: "id",
+  "filter[expedition_id]": options.expedition,
+}
+const qs_cores = {
+  name: "core",
   "per-page": -1,
   page: 1,
-  sort: "section",
-  "filter[core_id]": options.core_id,
+  sort: "id",
+  "filter[expedition_id]": options.expedition,
+}
+const qs_expedition = {
+  name: "expedition",
+  "filter[id]": options.expedition,
 }
 
-let split_sections_by_core = {}