so6_to_czml.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# so6 air traffic file to CesiumJS CZML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### <br> Build CZML from pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_start_stop(df):\n",
    "    starttime = str(min(df['date_begin_time_begin'])).replace(\" \", \"T\").replace(\".000\", \"Z\")\n",
    "    stoptime = str(max(df['date_end_time_end'])).replace(\" \", \"T\").replace(\".000\", \"Z\")\n",
    "    availability = starttime + \"/\" + stoptime\n",
    "    return (starttime, stoptime, availability)\n",
    "\n",
    "def get_4D_coords(group):\n",
    "    coords = []\n",
    "    t = 0\n",
    "    for _,row in group.iterrows():\n",
    "        coords.extend([t, row['lon_begin'], row['lat_begin'], row['fl_begin']])\n",
    "        t += row['duration']\n",
    "    return coords\n",
    "\n",
    "class CZMLBuilder:\n",
    "    \n",
    "    def __init__(self, df, global_id, global_name, global_version, global_author, \n",
    "                 default_time_multiplier=100, path_outline_width=3, path_width=3, \n",
    "                 path_lead_time=0, path_trail_time=100000, path_resolution=5,\n",
    "                 point_outline_width=2, point_pixel_size=6, \n",
    "                 point_height_reference=\"NONE\"):\n",
    "        \n",
    "        self.path_outline_width = path_outline_width\n",
    "        self.path_width = path_width\n",
    "        self.path_lead_time = path_lead_time\n",
    "        self.path_trail_time = path_trail_time\n",
    "        self.path_resolution = path_resolution\n",
    "        self.point_outline_width = point_outline_width\n",
    "        self.point_pixel_size = point_pixel_size\n",
    "        self.point_height_reference = point_height_reference\n",
    "        starttime, _, availability = get_start_stop(df)\n",
    "        self.df = df\n",
    "        self.groups = self.df.groupby('flight_identifier')\n",
    "        self.czml_output = []\n",
    "        \n",
    "        global_element = {\n",
    "                        \"id\": global_id, \n",
    "                        \"name\": global_name, \n",
    "                        \"version\": global_version, \n",
    "                        \"author\": global_author, \"clock\": {\"interval\": availability, \n",
    "                                                            \"currentTime\": starttime, \n",
    "                                                            \"multiplier\": default_time_multiplier\n",
    "                                                          }\n",
    "                        }\n",
    "        self.czml_output.append(global_element)\n",
    "    \n",
    "    def _get_path(self, group, path_id, starttime, availability, color):\n",
    "        path = {\n",
    "            \"id\": path_id,\n",
    "            \"availability\": availability,\n",
    "            \"position\": {\n",
    "                \"epoch\": starttime,\n",
    "                \"cartographicDegrees\": get_4D_coords(group)\n",
    "            },\n",
    "            \"path\": {\n",
    "                \"material\": {\n",
    "                    \"polylineOutline\": {\n",
    "                        \"color\": {\n",
    "                            \"rgba\": color\n",
    "                        },\n",
    "                        \"outlineColor\": {\n",
    "                            \"rgba\": color \n",
    "                        },\n",
    "                        \"outlineWidth\": self.path_outline_width\n",
    "                    }\n",
    "                },\n",
    "                \"width\": self.path_width,\n",
    "                \"leadTime\": self.path_lead_time,\n",
    "                \"trailTime\": self.path_trail_time,\n",
    "                \"resolution\": self.path_resolution\n",
    "            }\n",
    "        }\n",
    "        return path\n",
    "    \n",
    "    def _get_point(self, group, point_id, starttime, availability, color):\n",
    "        point = {\n",
    "            \"id\": point_id,\n",
    "            \"availability\": availability,\n",
    "\n",
    "            \"position\": {\n",
    "                \"epoch\": starttime,\n",
    "                \"cartographicDegrees\": get_4D_coords(group)\n",
    "            },\n",
    "            \"point\": {\n",
    "                \"color\": {\n",
    "                    \"rgba\": [255,255,255,200] # white center instead of color\n",
    "                },\n",
    "                \"outlineColor\": {\n",
    "                    \"rgba\": color\n",
    "                },\n",
    "                \"outlineWidth\": self.point_outline_width,\n",
    "                \"pixelSize\": self.point_pixel_size,\n",
    "                \"heightReference\": self.point_height_reference\n",
    "            }   \n",
    "        }\n",
    "        return point\n",
    "    \n",
    "    def save(self, file_path, path=True, point=True):\n",
    "        for name, group in self.groups:\n",
    "            color = [random.randint(0,255), random.randint(0,255), random.randint(0,255), 150]\n",
    "            starttime, _, availability = get_start_stop(group)\n",
    "            if path:\n",
    "                self.czml_output.append(self._get_path(group, name, starttime, availability, color))\n",
    "            if point:\n",
    "                self.czml_output.append(self._get_point(group, name, starttime, availability, color))\n",
    "        with open(file_path, 'w') as outfile:\n",
    "            json.dump(self.czml_output, outfile)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### <br> Load so6 file as a pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_begin_time_begin</th>\n",
       "      <th>date_end_time_end</th>\n",
       "      <th>flight_origin</th>\n",
       "      <th>flight_destination</th>\n",
       "      <th>aircraft_type</th>\n",
       "      <th>fl_begin</th>\n",
       "      <th>fl_end</th>\n",
       "      <th>status</th>\n",
       "      <th>callsign</th>\n",
       "      <th>lat_begin</th>\n",
       "      <th>lon_begin</th>\n",
       "      <th>lat_end</th>\n",
       "      <th>lon_end</th>\n",
       "      <th>flight_identifier</th>\n",
       "      <th>sequence</th>\n",
       "      <th>length</th>\n",
       "      <th>parity</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2018-03-01 14:25:00</td>\n",
       "      <td>2018-03-01 14:25:44</td>\n",
       "      <td>ENXU</td>\n",
       "      <td>ENXN</td>\n",
       "      <td>S92</td>\n",
       "      <td>60.96</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>HKS147</td>\n",
       "      <td>59.445833</td>\n",
       "      <td>2.344444</td>\n",
       "      <td>59.462500</td>\n",
       "      <td>2.365278</td>\n",
       "      <td>215700734</td>\n",
       "      <td>1</td>\n",
       "      <td>1.184730</td>\n",
       "      <td>0</td>\n",
       "      <td>44.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2018-03-01 06:41:00</td>\n",
       "      <td>2018-03-01 06:41:45</td>\n",
       "      <td>LEAL</td>\n",
       "      <td>EGPD</td>\n",
       "      <td>B738</td>\n",
       "      <td>30.48</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>RYR8005</td>\n",
       "      <td>38.282222</td>\n",
       "      <td>-0.558056</td>\n",
       "      <td>38.275278</td>\n",
       "      <td>-0.525556</td>\n",
       "      <td>215689750</td>\n",
       "      <td>1</td>\n",
       "      <td>1.586456</td>\n",
       "      <td>0</td>\n",
       "      <td>45.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2018-03-01 06:41:45</td>\n",
       "      <td>2018-03-01 06:41:55</td>\n",
       "      <td>LEAL</td>\n",
       "      <td>EGPD</td>\n",
       "      <td>B738</td>\n",
       "      <td>396.24</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>RYR8005</td>\n",
       "      <td>38.275278</td>\n",
       "      <td>-0.525556</td>\n",
       "      <td>38.273056</td>\n",
       "      <td>-0.514722</td>\n",
       "      <td>215689750</td>\n",
       "      <td>2</td>\n",
       "      <td>0.527418</td>\n",
       "      <td>0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2018-03-01 06:41:55</td>\n",
       "      <td>2018-03-01 06:42:03</td>\n",
       "      <td>LEAL</td>\n",
       "      <td>EGPD</td>\n",
       "      <td>B738</td>\n",
       "      <td>609.60</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>RYR8005</td>\n",
       "      <td>38.273056</td>\n",
       "      <td>-0.514722</td>\n",
       "      <td>38.270556</td>\n",
       "      <td>-0.503889</td>\n",
       "      <td>215689750</td>\n",
       "      <td>3</td>\n",
       "      <td>0.531892</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2018-03-01 06:42:03</td>\n",
       "      <td>2018-03-01 06:42:18</td>\n",
       "      <td>LEAL</td>\n",
       "      <td>EGPD</td>\n",
       "      <td>B738</td>\n",
       "      <td>609.60</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>RYR8005</td>\n",
       "      <td>38.270556</td>\n",
       "      <td>-0.503889</td>\n",
       "      <td>38.266111</td>\n",
       "      <td>-0.481944</td>\n",
       "      <td>215689750</td>\n",
       "      <td>4</td>\n",
       "      <td>1.067580</td>\n",
       "      <td>0</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  date_begin_time_begin   date_end_time_end flight_origin flight_destination  \\\n",
       "0   2018-03-01 14:25:00 2018-03-01 14:25:44          ENXU               ENXN   \n",
       "3   2018-03-01 06:41:00 2018-03-01 06:41:45          LEAL               EGPD   \n",
       "4   2018-03-01 06:41:45 2018-03-01 06:41:55          LEAL               EGPD   \n",
       "5   2018-03-01 06:41:55 2018-03-01 06:42:03          LEAL               EGPD   \n",
       "6   2018-03-01 06:42:03 2018-03-01 06:42:18          LEAL               EGPD   \n",
       "\n",
       "  aircraft_type  fl_begin  fl_end  status callsign  lat_begin  lon_begin  \\\n",
       "0           S92     60.96       1       2   HKS147  59.445833   2.344444   \n",
       "3          B738     30.48      13       0  RYR8005  38.282222  -0.558056   \n",
       "4          B738    396.24      20       0  RYR8005  38.275278  -0.525556   \n",
       "5          B738    609.60      20       2  RYR8005  38.273056  -0.514722   \n",
       "6          B738    609.60      25       0  RYR8005  38.270556  -0.503889   \n",
       "\n",
       "     lat_end   lon_end  flight_identifier  sequence    length  parity  \\\n",
       "0  59.462500  2.365278          215700734         1  1.184730       0   \n",
       "3  38.275278 -0.525556          215689750         1  1.586456       0   \n",
       "4  38.273056 -0.514722          215689750         2  0.527418       0   \n",
       "5  38.270556 -0.503889          215689750         3  0.531892       0   \n",
       "6  38.266111 -0.481944          215689750         4  1.067580       0   \n",
       "\n",
       "   duration  \n",
       "0      44.0  \n",
       "3      45.0  \n",
       "4      10.0  \n",
       "5       8.0  \n",
       "6      15.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "YOUR_SO6_FILE_PATH = 'your_so6_file.so6'\n",
    "columns = ['segment_identifier', 'flight_origin', 'flight_destination', 'aircraft_type', \n",
    "           'time_begin', 'time_end', 'fl_begin', 'fl_end', 'status', 'callsign', \n",
    "           'date_begin', 'date_end', 'lat_begin', 'lon_begin', 'lat_end', 'lon_end',\n",
    "           'flight_identifier', 'sequence', 'length', 'parity']\n",
    "parser = lambda x,y: pd.datetime.strptime(x + y, '%y%m%d%H%M%S')\n",
    "\n",
    "df = pd.read_csv(YOUR_SO6_FILE_PATH, sep=' ', header=None, names=columns, \n",
    "                 parse_dates=[['date_begin','time_begin'],['date_end','time_end']], date_parser=parser)\n",
    "\n",
    "df.query('length > 0', inplace=True) # filter null segments\n",
    "df['duration'] = (df['date_end_time_end'] - df['date_begin_time_begin']).dt.total_seconds()\n",
    "df['fl_begin'] = df['fl_begin'].apply(lambda x: x * 30.48) # FL to meters\n",
    "lat_lon_cols = ['lat_begin','lon_begin','lat_end','lon_end']\n",
    "df[lat_lon_cols] = df[lat_lon_cols].apply(lambda x: x / 60)\n",
    "df.drop(columns=['segment_identifier'], inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### <br> Filter dataframe (if needed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_begin_time_begin</th>\n",
       "      <th>date_end_time_end</th>\n",
       "      <th>flight_origin</th>\n",
       "      <th>flight_destination</th>\n",
       "      <th>aircraft_type</th>\n",
       "      <th>fl_begin</th>\n",
       "      <th>fl_end</th>\n",
       "      <th>status</th>\n",
       "      <th>callsign</th>\n",
       "      <th>lat_begin</th>\n",
       "      <th>lon_begin</th>\n",
       "      <th>lat_end</th>\n",
       "      <th>lon_end</th>\n",
       "      <th>flight_identifier</th>\n",
       "      <th>sequence</th>\n",
       "      <th>length</th>\n",
       "      <th>parity</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4501</th>\n",
       "      <td>2018-03-01 10:21:00</td>\n",
       "      <td>2018-03-01 10:21:20</td>\n",
       "      <td>LFBO</td>\n",
       "      <td>LFPO</td>\n",
       "      <td>A321</td>\n",
       "      <td>152.4</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>AFR81VM</td>\n",
       "      <td>43.635000</td>\n",
       "      <td>1.367778</td>\n",
       "      <td>43.626944</td>\n",
       "      <td>1.373889</td>\n",
       "      <td>215694215</td>\n",
       "      <td>1</td>\n",
       "      <td>0.551402</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4502</th>\n",
       "      <td>2018-03-01 10:21:20</td>\n",
       "      <td>2018-03-01 10:22:01</td>\n",
       "      <td>LFBO</td>\n",
       "      <td>LFPO</td>\n",
       "      <td>A321</td>\n",
       "      <td>304.8</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>AFR81VM</td>\n",
       "      <td>43.626944</td>\n",
       "      <td>1.373889</td>\n",
       "      <td>43.618889</td>\n",
       "      <td>1.380000</td>\n",
       "      <td>215694215</td>\n",
       "      <td>2</td>\n",
       "      <td>0.551420</td>\n",
       "      <td>0</td>\n",
       "      <td>41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4503</th>\n",
       "      <td>2018-03-01 10:22:01</td>\n",
       "      <td>2018-03-01 10:22:39</td>\n",
       "      <td>LFBO</td>\n",
       "      <td>LFPO</td>\n",
       "      <td>A321</td>\n",
       "      <td>609.6</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>AFR81VM</td>\n",
       "      <td>43.618889</td>\n",
       "      <td>1.380000</td>\n",
       "      <td>43.603056</td>\n",
       "      <td>1.392222</td>\n",
       "      <td>215694215</td>\n",
       "      <td>3</td>\n",
       "      <td>1.088311</td>\n",
       "      <td>0</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4504</th>\n",
       "      <td>2018-03-01 10:22:39</td>\n",
       "      <td>2018-03-01 10:22:58</td>\n",
       "      <td>LFBO</td>\n",
       "      <td>LFPO</td>\n",
       "      <td>A321</td>\n",
       "      <td>914.4</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>AFR81VM</td>\n",
       "      <td>43.603056</td>\n",
       "      <td>1.392222</td>\n",
       "      <td>43.594167</td>\n",
       "      <td>1.390556</td>\n",
       "      <td>215694215</td>\n",
       "      <td>4</td>\n",
       "      <td>0.538227</td>\n",
       "      <td>0</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4505</th>\n",
       "      <td>2018-03-01 10:22:58</td>\n",
       "      <td>2018-03-01 10:23:45</td>\n",
       "      <td>LFBO</td>\n",
       "      <td>LFPO</td>\n",
       "      <td>A321</td>\n",
       "      <td>1066.8</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>AFR81VM</td>\n",
       "      <td>43.594167</td>\n",
       "      <td>1.390556</td>\n",
       "      <td>43.576389</td>\n",
       "      <td>1.387222</td>\n",
       "      <td>215694215</td>\n",
       "      <td>5</td>\n",
       "      <td>1.076460</td>\n",
       "      <td>0</td>\n",
       "      <td>47.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     date_begin_time_begin   date_end_time_end flight_origin  \\\n",
       "4501   2018-03-01 10:21:00 2018-03-01 10:21:20          LFBO   \n",
       "4502   2018-03-01 10:21:20 2018-03-01 10:22:01          LFBO   \n",
       "4503   2018-03-01 10:22:01 2018-03-01 10:22:39          LFBO   \n",
       "4504   2018-03-01 10:22:39 2018-03-01 10:22:58          LFBO   \n",
       "4505   2018-03-01 10:22:58 2018-03-01 10:23:45          LFBO   \n",
       "\n",
       "     flight_destination aircraft_type  fl_begin  fl_end  status callsign  \\\n",
       "4501               LFPO          A321     152.4      10       0  AFR81VM   \n",
       "4502               LFPO          A321     304.8      20       0  AFR81VM   \n",
       "4503               LFPO          A321     609.6      30       0  AFR81VM   \n",
       "4504               LFPO          A321     914.4      35       0  AFR81VM   \n",
       "4505               LFPO          A321    1066.8      50       0  AFR81VM   \n",
       "\n",
       "      lat_begin  lon_begin    lat_end   lon_end  flight_identifier  sequence  \\\n",
       "4501  43.635000   1.367778  43.626944  1.373889          215694215         1   \n",
       "4502  43.626944   1.373889  43.618889  1.380000          215694215         2   \n",
       "4503  43.618889   1.380000  43.603056  1.392222          215694215         3   \n",
       "4504  43.603056   1.392222  43.594167  1.390556          215694215         4   \n",
       "4505  43.594167   1.390556  43.576389  1.387222          215694215         5   \n",
       "\n",
       "        length  parity  duration  \n",
       "4501  0.551402       0      20.0  \n",
       "4502  0.551420       0      41.0  \n",
       "4503  1.088311       0      38.0  \n",
       "4504  0.538227       0      19.0  \n",
       "4505  1.076460       0      47.0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.query('flight_origin == \"LFBO\"', inplace=True) # ex: only flights from LFBO\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### <br> Write the CZML file from the so6 dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "czml_builder = CZMLBuilder(df, \"document\", \"test_so6\", \"1.0\", \"your_name\")\n",
    "czml_builder.save('test.czml')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}