_PY07-LongMovementsPreProcessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating .pkl's containing only the dragging tasks "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 96 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import ast\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandarallel import pandarallel\n",
    "# Initialization\n",
    "pandarallel.initialize()\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import scipy\n",
    "from scipy import interpolate, stats\n",
    "\n",
    "%run py/constants.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sqrt_sum(tileX,tileY,targetX,targetY):\n",
    "    a = np.array([tileX,tileY])\n",
    "    b = np.array([targetX,targetY])\n",
    "    dist = np.sqrt(np.sum((a-b)**2, axis=0))       \n",
    "    return dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getFirstTime(timestamp):\n",
    "    if type(timestamp) is np.ndarray:\n",
    "        firstTime = timestamp[0]\n",
    "        return firstTime\n",
    "    else:\n",
    "        return timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getLastTime(timestamp):\n",
    "    if type(timestamp) is np.ndarray:\n",
    "        lastTime = timestamp[-1]\n",
    "        return lastTime\n",
    "    else:\n",
    "        return timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def applyChanges(f, np_arr):\n",
    "    #print(len(np_arr))\n",
    "    np_arr = list(np_arr)\n",
    "    np_arr = list(map(lambda x: x + np.round(f(x)), np_arr))\n",
    "    np_arr = list(map(lambda x: x.astype(int), np_arr))\n",
    "    return np_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3 288 384\n",
      "S4 288 384\n",
      "OPO 288 384\n",
      "N6 288 384\n",
      "CPU times: user 974 ms, sys: 0 ns, total: 974 ms\n",
      "Wall time: 972 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "task = \"DRAG\"\n",
    "\n",
    "phone_dict = {\"S3\":[], \"S4\":[], \"OPO\":[], \"N6\":[]}\n",
    "\n",
    "for path, subdirs, files in os.walk(\"./raw_data_phone/\"):\n",
    "    for file in files:\n",
    "        file_path = os.path.join(path, file)\n",
    "        \n",
    "        if \"StudyRun\" not in file or task not in file:\n",
    "            continue\n",
    "            \n",
    "        pid, phone, cond, task = file.replace(\"StudyRun\",\"\").replace(\".txt\",\"\").split(\"_\")\n",
    "        pid = int(pid)\n",
    "        device = conv_phone_name(phone)\n",
    "        \n",
    "        df = pd.read_csv(file_path, sep=\";\")\n",
    "        df[\"Phone\"] = device\n",
    "        df[\"Participant\"] = pid\n",
    "        df[\"Cond\"] = cond\n",
    "        phone_dict[device].append(df)\n",
    "\n",
    "for phone, df_dict in phone_dict.items():\n",
    "    phone_dict[phone] = pd.concat(df_dict)\n",
    "    \n",
    "\n",
    "for phone, phones_df in phone_dict.items():\n",
    "    phones_df[\"Distance\"] =  phones_df.apply(lambda row: (sqrt_sum(row['tileX'],row['tileY'],row['targetX'],row['targetY'])*get_pixelsize_for_device(row['Phone']))/10, axis=1)\n",
    "    phones_df = phones_df.reset_index(drop=True)\n",
    "    phones_df = phones_df.sort_values(by=[\"Distance\"])\n",
    "    print(phone,int(len(phones_df)*0.75), len(phones_df))\n",
    "    #phones_df = phones_df[int(len(phones_df)*0.75):]\n",
    "    phones_df.timestamp = phones_df.timestamp.apply(lambda x: np.array(ast.literal_eval(x)))\n",
    "    phones_df['FirstTime'] = phones_df.apply(lambda row: getFirstTime(row['timestamp']), axis=1) \n",
    "    phones_df['LastTime'] = phones_df.apply(lambda row: getLastTime(row['timestamp']), axis=1)\n",
    "    phone_dict[phone] = phones_df.sort_values(by=[\"LastTime\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def syncS3Times(pid, c, df):\n",
    "    filepath = \"./raw_data_phone/timestamp_adjusted/timestamp_matching_s3_P%s_%s.txt\"%(str(pid),c)\n",
    "    #all S3 files for participant pid combined\n",
    "    concatDf = df\n",
    "    #UDP - file with PC timestamps and related phonetimestamps\n",
    "    matching_df = pd.read_csv(filepath, delimiter =\",\")\n",
    "    #name the columns\n",
    "    matching_df= matching_df.rename(index=str , columns = {matching_df.columns[0]:\"Phonestamp\",matching_df.columns[1]:\"Motivestamp\"})\n",
    "    #get difference in ms\n",
    "    matching_df[\"diff\"] = matching_df[\"Motivestamp\"] - matching_df[\"Phonestamp\"]\n",
    "    #matching_df = \n",
    "    #merge the matching df with S3 df (Phonestamp X LastTime)\n",
    "    merged_df = pd.merge_asof(matching_df, concatDf, left_on='Phonestamp',right_on='LastTime', direction = 'nearest')\n",
    "\n",
    "    '''\n",
    "    create df that holds X and Y axis for the 1D interpolation \n",
    "    (append the first diff and last diff for border case)\n",
    "    '''\n",
    "    interpol_df = pd.DataFrame(data={\"interpolTime\":merged_df[\"Phonestamp\"],\"diff\": merged_df[\"diff\"]}).append(\n",
    "        {\"diff\":merged_df[\"diff\"].iloc[0],\"interpolTime\":0}, ignore_index=True).append(\n",
    "        {\"diff\":merged_df[\"diff\"].iloc[-1],\"interpolTime\":1600000000000}, ignore_index=True).sort_values(by=['interpolTime'])\n",
    "\n",
    "    #interpolate along the X(timestamps) and Y(diff) axes \n",
    "    f = interpolate.interp1d(interpol_df[\"interpolTime\"], interpol_df [\"diff\"])\n",
    "    #add new column that holds interpolated values \n",
    "    concatDf[\"interpol\"] = applyChanges(f, concatDf[\"timestamp\"])\n",
    "    concatDf['FirstTime'] = concatDf.apply(lambda row: getFirstTime(row['interpol']), axis=1) \n",
    "    concatDf['LastTime'] = concatDf.apply(lambda row: getLastTime(row['interpol']), axis=1)\n",
    "\n",
    "    return concatDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getDraggingTimes(pid,phone,cond):\n",
    "    df = phone_dict[phone]\n",
    "    df = df[(df.Phone==phone)&(df.Cond == cond)&(df.Participant == pid)].copy(deep=True)\n",
    "    if (len(df)==0):\n",
    "        return None,None\n",
    "    if phone == \"S3\":\n",
    "        df = syncS3Times(pid, cond, df)\n",
    "    return df.FirstTime.tolist(), df.LastTime.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def doJob(pickle):\n",
    "    i = 0\n",
    "    path = \"./TransformedPickles/\"\n",
    "    pid,phone,cond = pickle.replace(\".pkl\",\"\").split(\"_\")\n",
    "    pid = int(pid.replace(\"P\",\"\"))\n",
    "    filepath = path+pickle\n",
    "    print(\"Working on file %s\"%(filepath))\n",
    "    firstTimes, lastTimes = getDraggingTimes(pid,phone,cond)\n",
    "    if firstTimes == None:\n",
    "        return\n",
    "    df = pd.read_pickle(filepath)\n",
    "    df = df[df.Task == \"Fitts\"].copy(deep=True)\n",
    "    df[\"Drag\"] = None\n",
    "    for firstTime, lastTime in zip(firstTimes,lastTimes):\n",
    "        df.loc[(df.Time>=firstTime)&(df.Time<=lastTime),'Drag'] = 'P%i_%i' % (pid, i)\n",
    "        i+=1\n",
    "    df = df.dropna(subset=[\"Drag\"])\n",
    "    #get list with first and lasttimes of longest movements that were calculated before\n",
    "    #print(firstTimes,lastTimes)\n",
    "    pickle_path = \"./DraggingPickles/\"\n",
    "    pickle_name = 'P%i'%pid\n",
    "    pickle_path = pickle_path+pickle_name + (\"_%s_%s.pkl\"%(phone, cond))\n",
    "    df.to_pickle(pickle_path)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = []\n",
    "for pickle in os.listdir(\"./TransformedPickles/\"):\n",
    "    if \"checkpoints\" in pickle:\n",
    "        continue\n",
    "    files.append(pickle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on file ./TransformedPickles/P3_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P5_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P14_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P10_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P6_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P11_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P8_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P10_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P21_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P19_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P19_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P7_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P7_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P12_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P10_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P12_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P6_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P7_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P12_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P7_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P13_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P20_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P6_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P5_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P16_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P11_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P21_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P10_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P3_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P3_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P21_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P20_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P17_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P6_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P8_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P3_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P4_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P21_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P11_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P5_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P12_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P5_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P13_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P13_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P10_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P11_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P19_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P13_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P20_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P12_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P14_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P10_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P6_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P16_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P14_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P14_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P8_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P13_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P3_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P7_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P20_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P10_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P20_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P13_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P8_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P16_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P4_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P14_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P4_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P14_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P7_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P20_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P17_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P8_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P4_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P4_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P7_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P6_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P5_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P4_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P19_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P12_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P13_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P16_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P4_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P11_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P6_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P17_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P16_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P19_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P17_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P14_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P5_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P16_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P17_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P8_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P14_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P5_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P10_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P21_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P12_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P17_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P21_S3_walking.pkl\n",
      "Working on file ./TransformedPickles/P12_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P17_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P6_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P3_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P20_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P21_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P3_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P19_N6_seated.pkl\n",
      "Working on file ./TransformedPickles/P19_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P16_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P13_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P7_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P4_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P3_S3_seated.pkl\n",
      "Working on file ./TransformedPickles/P11_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P21_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P11_N6_walking.pkl\n",
      "Working on file ./TransformedPickles/P19_OPO_seated.pkl\n",
      "Working on file ./TransformedPickles/P5_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P8_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P17_S4_walking.pkl\n",
      "Working on file ./TransformedPickles/P8_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P11_S4_seated.pkl\n",
      "Working on file ./TransformedPickles/P20_OPO_walking.pkl\n",
      "Working on file ./TransformedPickles/P16_S4_walking.pkl\n",
      "CPU times: user 241 ms, sys: 748 ms, total: 988 ms\n",
      "Wall time: 3.33 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "dfFiles = pd.DataFrame(files)\n",
    "dfFiles.columns=[\"File\"]\n",
    "ret = dfFiles.File.parallel_apply(lambda x: doJob(x))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}