initial commit

.gitignore

+# Ignore plots
+*.png
+
+# Ignore IDE files
+.idea/
+
+# Python
+venv/
\ No newline at end of file

.gitlab-ci.yml

+image: python:3
+
+stages:
+  - test
+
+before_script:
+  - pip install -r requirements.txt
+  - pip install -r requirements-dev.txt
+
+test:pylint:
+  stage: test
+  script:
+    - pipenv run pylint --rcfile test/linting/pylintrc src/*.py
+  only:
+    changes:
+      - "**/*.py"
+      - "test/linting/pylintrc"
+      - ".gitlab-ci.yml"
+
+test:unittest:
+  stage: test
+  script:
+    - pipenv run python -m unittest discover test/unittest
+  only:
+    changes:
+      - "**/*.py"
+      - ".gitlab-ci.yml"

README.md

+# Astronaut Analysis
+
+The script analyzes publicly available astronauts data from [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page).
+It generates a set of plots focusing on aspects such as time humans spent in space, the gender distribution as well as the age distribution.
+
+## Install
+
+The script comes with a predefined Python environment, which is managed by [pipenv](https://github.com/pypa/pipenv).
+The environment handles all dependencies.
+
+> The script has been successfully tested on 5.7.8-arch1-1 with Python 3.8.3
+
+Please clone this repository and install the [dependencies](requirements.txt) as follows:
+
+```bash
+git clone ...
+cd astronaut-analysis
+pip install -r requirements.txt
+```
+
+## Usage
+
+You can run the script as follows:
+
+```bash
+python src/astronaut-analysis.py
+```
+
+The script processes the [astronauts data set]( data/astronauts.json) and stores the plots in the directory `results`.
+The directory will be created by the script.
+Existing result plots will be overwritten.
+
+### Astronaut Data
+
+The data set has been generated from the following SPARQL query [[1]] (retrieval date: 2018-10-25).
+
+You can replace the data set as follows:
+- Run the SPARQL query
+- Download the resulting data formatted as JSON
+- Replace the file `data/astronauts.json`
+
+[1]: https://query.wikidata.org/#%23Birthplaces%20of%20astronauts%0ASELECT%20DISTINCT%20%3Fastronaut%20%3FastronautLabel%20%3Fbirthdate%20%3FbirthplaceLabel%20%3Fsex_or_genderLabel%20%3Ftime_in_space%20%3Fdate_of_death%20WHERE%20%7B%0A%20%20%3Fastronaut%20%3Fx1%20wd%3AQ11631.%0A%20%20%3Fastronaut%20wdt%3AP569%20%3Fbirthdate.%0A%20%20%3Fastronaut%20wdt%3AP19%20%3Fbirthplace.%0A%20%20SERVICE%20wikibase%3Alabel%20%7B%20bd%3AserviceParam%20wikibase%3Alanguage%20%22en%22.%20%7D%0A%20%20OPTIONAL%20%7B%20%3Fastronaut%20wdt%3AP21%20%3Fsex_or_gender.%20%7D%0A%20%20OPTIONAL%20%7B%20%3Fastronaut%20wdt%3AP2873%20%3Ftime_in_space.%20%7D%0A%20%20OPTIONAL%20%7B%20%3Fastronaut%20wdt%3AP570%20%3Fdate_of_death.%20%7D%0A%7D%0AORDER%20BY%20DESC%28%3Ftime_in_space%29

requirements-dev.txt

+pylint
\ No newline at end of file

requirements.txt

+pandas==1.0.5
+matplotlib==3.2.2
\ No newline at end of file

src/astronauts_analysis.py

+"""
+This script analysis a data set about astronauts and creates different
+plots as result.
+"""
+
+from datetime import date
+from os import makedirs
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+plt.style.use("ggplot")
+_ASTRONAUT_DATA = "data/astronauts.json"
+_OUTPUT_PATH = "results"
+
+
+##
+# Data preparation functions
+##
+def prepare_data_set(data_frame: pd.DataFrame) -> pd.DataFrame:
+    """
+    Prepares the raw data by:
+        - dropping NaN's
+        - setting data types
+        - calculating some extra columns
+
+    Args:
+       data_frame: A pandas DataFrame.
+
+    Returns:
+        A pandas DataFrame with preprocessed data.
+    """
+    data_frame = rename_columns(data_frame)
+    data_frame = data_frame.set_index("astronaut_id")
+
+    # Set pandas dtypes for columns with date or time
+    data_frame = data_frame.dropna(subset=["time_in_space"])
+    data_frame["time_in_space"] = data_frame["time_in_space"].astype(int)
+    data_frame["time_in_space"] = pd.to_timedelta(data_frame["time_in_space"], unit="m")
+    data_frame["birthdate"] = pd.to_datetime(data_frame["birthdate"])
+    data_frame["date_of_death"] = pd.to_datetime(data_frame["date_of_death"])
+    data_frame.sort_values("birthdate", inplace=True)
+
+    # Calculate extra columns from the original data
+    data_frame["time_in_space_D"] = data_frame["time_in_space"].astype("timedelta64[D]")
+    data_frame["alive"] = data_frame["date_of_death"].apply(is_alive)
+    data_frame["age"] = data_frame["birthdate"].apply(calculate_age)
+    data_frame["died_with_age"] = data_frame.apply(died_with_age, axis=1)
+    return data_frame
+
+
+def rename_columns(data_frame):
+    """
+    The original column naming in the data set is not useful
+    for programming with pandas. So we rename it.
+    """
+
+    name_mapping = {
+        "astronaut": "astronaut_id",
+        "astronautLabel": "name",
+        "birthplaceLabel": "birthplace",
+        "sex_or_genderLabel": "sex_or_gender",
+    }
+    data_frame = data_frame.rename(index=str, columns=name_mapping)
+    return data_frame
+
+
+def is_alive(date_of_death) -> bool:
+    """
+    Checks, if 'date_of_death' exists or not.
+
+    Args:
+       date_of_death: Either a pandas NaTType or a pandas Timestamp.
+
+    Returns:
+        bool
+    """
+    if pd.isnull(date_of_death):
+        return True
+    return False
+
+
+def calculate_age(born: pd.Timestamp) -> int:
+    """
+    Calculates an age from a date.
+
+    Args:
+       born: pandas.Timestamp
+
+    Returns:
+        int
+    """
+
+    if not isinstance(born, pd.Timestamp):
+        raise TypeError(f'expected {pd.Timestamp}, got {type(born)}')
+
+    today = date.today()
+    return today.year - born.year - ((today.month, today.day) < (born.month, born.day))
+
+
+def died_with_age(row: pd.Series):
+    """
+    Calculates an age from a birthdate and date_of_death.
+
+    Args:
+       row: pandas.Series with the columns `birthdate` and `date_of_death`
+
+    Returns:
+        int
+    """
+    if pd.isnull(row["date_of_death"]):
+        return None
+    born = row["birthdate"]
+    today = row["date_of_death"]
+    return today.year - born.year - ((today.month, today.day) < (born.month, born.day))
+
+
+##
+# Plot functions
+##
+def create_time_of_x_in_space(data_frame, filename, title):
+    """
+    This function generated a plot with the summed up time of 'living beings'
+    in space over the years by their birthday's.
+    """
+
+    reduced_data_frame = data_frame[["birthdate", "time_in_space", "time_in_space_D"]].copy()
+    reduced_data_frame["accumulated_time_in_minutes"] = reduced_data_frame["time_in_space"].cumsum()
+    reduced_data_frame["accumulated_time_in_days"] = reduced_data_frame["time_in_space_D"].cumsum()
+    axs = reduced_data_frame.plot(x="birthdate", y="accumulated_time_in_days")
+    axs.set_title(title)
+    axs.set_xlabel("Years ")
+    axs.set_ylabel("t in days")
+    save(axs.get_figure(), filename)
+
+
+def create_age_histogram(age_data_frame, died_data_frame):
+    """
+    The function generates a combined histogram of astronauts
+    in the categories 'age at dead' and 'age alive'.
+    """
+
+    fig, axs = plt.subplots(1, 1)
+    axs.hist(
+        [died_data_frame["died_with_age"], age_data_frame["age"]],
+        bins=70,
+        range=(31, 100),
+        stacked=True,
+    )
+    axs.set_xlabel("Age")
+    axs.set_ylabel("Number of astronauts")
+    axs.set_title("Dead vs. Alive astronauts")
+    save(fig, "combined_histogram.png")
+
+
+def create_age_boxplot(age_data_frame, died_data_frame):
+    """
+    The function generates a boxplot of astronauts age distribution
+    in the categories dead and alive.
+    """
+
+    fig, axs = plt.subplots(1, 1)
+    axs.boxplot([died_data_frame["died_with_age"], age_data_frame["age"]])
+    axs.set_title("Age distribution; Dead vs. Alive astronauts")
+    axs.set_xlabel("Category")
+    plt.setp(axs, xticks=[1, 2], xticklabels=["Dead", "Alive"])
+    axs.set_ylabel("Age")
+    save(fig, "boxplot.png")
+
+
+def save(fig: plt.Figure, filename: str):
+    """
+    Saves a matplotlib Figure to a file. It overwrites existing files with the same filename.
+
+    Args:
+        fig: matplotlib.pyplot.Figure
+        filename: str
+    """
+    fig.savefig(Path(_OUTPUT_PATH).resolve() / Path(filename))
+
+
+def perform_analysis():
+    """ Glues data preparation and plotting. """
+
+    # Set up directory structure and preprocess data
+    makedirs(_OUTPUT_PATH, exist_ok=True)
+    data_frame = pd.read_json(Path(_ASTRONAUT_DATA).resolve())
+    data_frame = prepare_data_set(data_frame)
+
+    # Male humans in space
+    data_frame_male = data_frame.loc[
+        data_frame["sex_or_gender"] == "male", ["birthdate", "time_in_space", "time_in_space_D"]
+    ].copy()
+    create_time_of_x_in_space(
+        data_frame_male,
+        "male_humans_in_space.png",
+        "Total time male humans have spend in space",
+    )
+
+    # Female humans in space
+    data_frame_female = data_frame.loc[
+        data_frame["sex_or_gender"] == "female",
+        ["birthdate", "time_in_space", "time_in_space_D"],
+    ].copy()
+    create_time_of_x_in_space(
+        data_frame_female,
+        "female_humans_in_space.png",
+        "Total time female humans have spend in space",
+    )
+
+    # Humans in space
+    create_time_of_x_in_space(
+        data_frame, "humans_in_space.png", "Total time humans have spend in space"
+    )
+
+    # Dead and alive astronauts analysis
+    died_data_frame = data_frame.loc[data_frame["alive"] == 0, ["died_with_age"]].copy()
+    age_data_frame = data_frame.loc[data_frame["alive"] == 1, ["age"]].copy()
+
+    # Combined histogram of dead and alive astronauts
+    create_age_histogram(age_data_frame, died_data_frame)
+
+    # Box plots of dead vs alive astronauts
+    create_age_boxplot(age_data_frame, died_data_frame)
+
+
+# Main entry point
+if __name__ == "__main__":
+    perform_analysis()

test/linting/pylintrc

+[MASTER]
+
+# Specify a score threshold to be exceeded before program exits with error.
+# max = 10
+fail-under=9
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=0
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes

test/unittest/test_data_types.py

+import unittest
+import pandas as pd
+from src import astronauts_analysis
+
+
+class TestCalculateAge(unittest.TestCase):
+
+    def test_return_type(self):
+        birth_date = pd.Timestamp('1950-01-01')
+        self.assertEqual(type(astronauts_analysis.calculate_age(birth_date)), int)
+
+    def test_arg_type(self):
+        birth_date = '1950-01-01'
+        with self.assertRaises(TypeError):
+            astronauts_analysis.calculate_age(birth_date)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unittest/test_file_io.py

+import unittest
+import os
+from src import astronauts_analysis
+import shutil
+import time
+from datetime import datetime
+
+
+class TestDataRead(unittest.TestCase):
+
+    def test_no_data(self):
+        # set wrong data path
+        astronauts_analysis._ASTRONAUT_DATA = "data/astronauts.jsson"
+
+        with self.assertRaises(ValueError):
+            astronauts_analysis.perform_analysis()
+
+        # restore correct data path
+        astronauts_analysis._ASTRONAUT_DATA = "data/astronauts.json"
+
+
+class TestResultWrite(unittest.TestCase):
+
+    def test_output_files_exist(self):
+        astronauts_analysis.perform_analysis()
+
+        self.assertTrue(os.path.isfile('results/boxplot.png'))
+        self.assertTrue(os.path.isfile('results/combined_histogram.png'))
+        self.assertTrue(os.path.isfile('results/female_humans_in_space.png'))
+        self.assertTrue(os.path.isfile('results/humans_in_space.png'))
+        self.assertTrue(os.path.isfile('results/male_humans_in_space.png'))
+
+    def test_output_files_overwritten(self):
+        # remove previous generated outputs
+        shutil.rmtree('results')
+
+        # generate output
+        astronauts_analysis.perform_analysis()
+
+        # wait certain time, to get a good delta to previous edits
+        wait_seconds = 10
+        time.sleep(wait_seconds)
+
+        # overwrite previous generated output
+        astronauts_analysis.perform_analysis()
+
+        # calculate how much time passed since the output was generated
+        timedelta = datetime.utcnow() - datetime.utcfromtimestamp(os.path.getmtime('results/boxplot.png'))
+
+        # test that time passed < wait time
+        self.assertLess(timedelta.total_seconds(), wait_seconds)
+
+
+def preparations():
+    # remove previous generated results
+    shutil.rmtree('results')
+
+
+if __name__ == '__main__':
+    preparations()
+    unittest.main()