#!/usr/bin/env python3
""" amdgpu-plot  -  Plot GPU parameter curves

    A utility to continuously plot the trend of critical GPU parameters for all compatible
    AMD GPUs. The *--sleep N* can be used to specify the update interval.  The *amdgpu-plot*
    utility has 2 modes of operation.  The default mode is to read the GPU driver details
    directly, which is useful as a standalone utility.  The *--stdin* option causes
    *amdgpu-plot* to read GPU data from stdin.  This is how *amdgpu-monitor* produces the
    plot and can also be used to pipe your own data into the process.  The *--simlog*
    option can be used with the *--stdin* when a monitor log file is piped as stdin.
    This is useful for troubleshooting and can be used to display saved log results.
    The *--ltz* option results in the use of local time instead of UTC.  If you plan
    to run both *amdgpu-plot* and *amdgpu-monitor*, then the *--plot* option of the
    *amdgpu-monitor* utility should be used instead of both utilities in order reduce
    data reads by a factor of 2.

    Copyright (C) 2019  RueiKe

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""
__author__ = 'RueiKe'
__copyright__ = 'Copyright (C) 2019 RueiKe'
__credits__ = ['Craig Echt - Testing, Debug, Verification, and Documentation']
__license__ = 'GNU General Public License'
__program_name__ = 'amdgpu-plot'
__version__ = 'v2.6.0'
__maintainer__ = 'RueiKe'
__status__ = 'Stable Release'

import sys
import gc as garbcollect
import argparse
import re
import threading
import os
import time
from pathlib import Path
import numpy as np

try:
    from matplotlib.ticker import AutoLocator
    from matplotlib.backends.backend_gtk3cairo import FigureCanvasGTK3Cairo as FigureCanvas
    from matplotlib.figure import Figure
    import matplotlib.pyplot as plt
    #import matplotlib.dates as mdates
except ModuleNotFoundError as error:
    print('matplotlib import error: {}'.format(error))
    print('matplotlib is required for %s', __program_name__)
    print('Use \'sudo apt-get install python3-matplotlib\' to install')
    sys.exit(0)

try:
    import pandas as pd
except ModuleNotFoundError as error:
    print('Pandas import error: {}'.format(error))
    print('Pandas is required for %s', __program_name__)
    print('Install pip3 if needed: \'sudo apt install python3-pip\'')
    print('Then pip install pandas: \'pip3 install pandas\'')
    sys.exit(0)
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

try:
    import gi
except ModuleNotFoundError as error:
    print('gi import error: {}'.format(error))
    print('gi is required for %s', __program_name__)
    print('   In a venv, first install vext:  pip install --no-cache-dir vext')
    print('   Then install vext.gi:  pip install --no-cache-dir vext.gi')
    sys.exit(0)
gi.require_version('Gtk', '3.0')
from gi.repository import GLib, Gtk, Gdk

from GPUmodules import GPUmodule as GPU
from GPUmodules import env


# SEMAPHORE ############
pd_sem = threading.Semaphore()
########################


def hex_to_rgba(value):
    # Code copied from Stack Overflow
    value = value.lstrip('#')
    if len(value) == 3:
        value = ''.join([v*2 for v in list(value)])
    (r1, g1, b1, a1) = tuple(int(value[i:i+2], 16) for i in range(0, 6, 2))+(1,)
    (r1, g1, b1, a1) = (r1/255.00000, g1/255.00000, b1/255.00000, a1)
    return r1, g1, b1, a1


def get_stack_size():
    """Get stack size for caller's frame.

       Code copied from Stack Overflow
    """
    size = 2  # current frame and caller's frame always exist
    while True:
        try:
            sys._getframe(size)
            size += 1
        except ValueError:
            return size - 1  # subtract current frame


class PlotData:
    def __init__(self):
        self.df = pd.DataFrame()
        self.gui_comp = None
        self.gui_ready = False
        self.length = 200
        self.quit = False
        self.writer = False
        self.reader = False
        self.consec_writer = 0
        self.consec_reader = 0
        self.gpu_list = ''
        self.num_gpus = 1
        self.com_gpu_list = GPU.GPU_LIST()

    def set_gpus(self):
        self.num_gpus = self.df['Card#'].nunique()
        self.gpu_list = self.df['Card#'].unique()

    def get_plot_data(self):
        # get deep copy of plot data df
        # This may have contention issues
        # SEMAPHORE ############
        pd_sem.acquire()
        ########################
        ndf = self.df.copy()
        # SEMAPHORE ############
        pd_sem.release()
        ########################
        return ndf

    def kill_thread(self):
        self.reader = False
        self.quit = True
        print('Stopping reader thread')
        time.sleep(0.2)


class GuiComponents:
    def __init__(self, plot_data):
        plot_data.gui_comp = self
        self.ready = False
        self.gpu_list = plot_data.gpu_list
        self.num_gpus = plot_data.num_gpus
        self.gui_components = {}
        self.gpu_color = {}
        self.colors = {'plotface': '#404040', 'figface': '#909090', 'sclk_f': '#BED661', 'mclk_f': '#89E894',
                       'loading': '#1E90FF', 'power': '#E12B06', 'power_cap': '#800000', 'vddgfx': '#778899',
                       'temp': '#E0E0E0'}
        self.font_colors = {'plotface': '#000000', 'figface': '#000000', 'sclk_f': '#000000', 'mclk_f': '#000000',
                            'loading': '#FFFFFF', 'power': '#FFFFFF', 'power_cap': '#FFFFFF', 'vddgfx': '#000000',
                            'temp': '#000000'}
        gpu_color_list = ['#B52735', '#EBB035', '#06A2CB', '#218559', '#D0C6B1', '#E18A07', '#336688', '#7C821E']
        plot_item_list = ['loading', 'power', 'power_cap', 'temp', 'vddgfx', 'sclk_f', 'mclk_f']

        self.plot_items = {'loading': True, 'power': True, 'power_cap': True,
                           'temp': True, 'vddgfx': True,'sclk_f': True, 'mclk_f': True}

        self.gui_components['info_bar'] = {}
        self.gui_components['legend'] = {}
        self.gui_components['legend']['buttons'] = {}
        self.gui_components['legend']['plot_items'] = {}
        for plotitem in plot_item_list:
            self.gui_components['legend']['plot_items'][plotitem] = True
        self.gui_components['sclk_pstate_status'] = {}
        self.gui_components['sclk_pstate_status']['df_name'] = 'sclk_ps'
        self.gui_components['mclk_pstate_status'] = {}
        self.gui_components['mclk_pstate_status']['df_name'] = 'mclk_ps'
        self.gui_components['temp_status'] = {}
        self.gui_components['temp_status']['df_name'] = 'temp'
        self.gui_components['card_plots'] = {}
        for i, gpu in enumerate(self.gpu_list):
            self.gui_components['card_plots'][gpu] = {}
            self.gui_components['card_plots'][gpu]['color'] = gpu_color_list[i]
            self.gpu_color[gpu] = gpu_color_list[i]
        return
    
    def set_ready(self, mode):
        self.ready = mode

    def is_ready(self):
        return self.ready


class GPUPlotWindow(Gtk.Window):
    def __init__(self, gc, plot_data):
        box_spacing_val = 5
        num_bar_plots = 3
        if gc.num_gpus > 4:
            def_gp_y_size = 150
            def_bp_y_size = 200
        elif gc.num_gpus == 4:
            def_gp_y_size = 200
            def_bp_y_size = 200
        else:
            def_gp_y_size = 250
            def_bp_y_size = 250
        def_gp_x_size = 600
        def_bp_x_size = 250
        def_lab_y_size = 28
        if gc.num_gpus > num_bar_plots:
            tot_y_size = gc.num_gpus * (def_gp_y_size + def_lab_y_size)
            gp_y_size = def_gp_y_size
            bp_y_size = (tot_y_size - (num_bar_plots * def_lab_y_size))/num_bar_plots
        elif gc.num_gpus < num_bar_plots:
            tot_y_size = num_bar_plots * (def_bp_y_size + def_lab_y_size)
            bp_y_size = def_bp_y_size
            gp_y_size = (tot_y_size - (gc.num_gpus * def_lab_y_size))/gc.num_gpus
        else:
            gp_y_size = def_gp_y_size
            bp_y_size = def_bp_y_size

        Gtk.Window.__init__(self, title='amdgpu-plot')
        self.set_border_width(1)
        icon_file = os.path.join(env.gut_const.PATH, 'icons', 'amdgpu-plot.icon.png')
        if os.path.isfile(icon_file):
            self.set_icon_from_file(icon_file)
        grid = Gtk.Grid()
        grid.override_background_color(Gtk.StateType.NORMAL, Gdk.RGBA(0.7, 0.7, 0.7, 1))
        self.add(grid)

        # Get deep copy of current df
        ldf = plot_data.get_plot_data()

        row = 0
        # Top Bar - info
        gc.gui_components['info_bar']['gtk_obj'] = Gtk.Label()
        gc.gui_components['info_bar']['gtk_obj'].set_markup('<big><b>' + 'amdgpu-util Plot' + '</b>' + '</big>')
        gc.gui_components['info_bar']['gtk_obj'].override_color(Gtk.StateFlags.NORMAL, Gdk.RGBA(1.0, 1.0, 1.0, 1.0))
        gc.gui_components['info_bar']['gtk_obj'].set_property('margin-top', 1)
        gc.gui_components['info_bar']['gtk_obj'].set_property('margin-bottom', 1)
        gc.gui_components['info_bar']['gtk_obj'].set_property('margin-right', 4)
        gc.gui_components['info_bar']['gtk_obj'].set_property('margin-left', 4)
        gc.gui_components['info_bar']['gtk_obj'].set_alignment(0.5, 0.5)
        lbox = Gtk.Box(spacing=box_spacing_val)
        lbox.override_background_color(Gtk.StateType.NORMAL, Gdk.RGBA(.20, .40, .60, 1.0))
        lbox.set_property('margin-top', 1)
        lbox.set_property('margin-bottom', 1)
        lbox.set_property('margin-right', 1)
        lbox.set_property('margin-left', 1)
        lbox.pack_start(gc.gui_components['info_bar']['gtk_obj'], True, True, 0)
        grid.attach(lbox, 1, row, 4, 1)
        row += 1

        # Legend
        gc.gui_components['legend']['gtk_obj'] = Gtk.Label()
        gc.gui_components['legend']['gtk_obj'].set_markup('<big><b>' + 'Plot Items' + '</b>' + '</big>')
        gc.gui_components['legend']['gtk_obj'].override_color(Gtk.StateFlags.NORMAL, Gdk.RGBA(1.0, 1.0, 1.0, 1.0))
        gc.gui_components['legend']['gtk_obj'].set_property('margin-top', 1)
        gc.gui_components['legend']['gtk_obj'].set_property('margin-bottom', 1)
        gc.gui_components['legend']['gtk_obj'].set_property('margin-right', 4)
        gc.gui_components['legend']['gtk_obj'].set_property('margin-left', 4)
        gc.gui_components['legend']['gtk_obj'].set_alignment(0.5, 0.5)
        lbox = Gtk.Box(spacing=box_spacing_val)
        lbox.override_background_color(Gtk.StateType.NORMAL, Gdk.RGBA(.40, .40, .40, 1.0))
        lbox.set_property('margin-top', 1)
        lbox.set_property('margin-bottom', 1)
        lbox.set_property('margin-right', 1)
        lbox.set_property('margin-left', 1)
        lbox.pack_start(gc.gui_components['legend']['gtk_obj'], True, True, 0)
        for k, v in gc.gui_components['legend']['plot_items'].items():
            but_color = hex_to_rgba(gc.colors[k])
            but_font_color = hex_to_rgba(gc.font_colors[k])
            gc.gui_components['legend']['buttons'][k] = Gtk.Button('')
            for child in gc.gui_components['legend']['buttons'][k].get_children():
                child.set_label('<big><b>' + k + '</b></big>')
                child.set_use_markup(True)
                child.override_color(
                        Gtk.StateFlags.NORMAL,
                        Gdk.RGBA(but_font_color[0], but_font_color[1], but_font_color[2], but_font_color[3]))
                child.override_background_color(
                        Gtk.StateFlags.NORMAL,
                        Gdk.RGBA(but_color[0], but_color[1], but_color[2], but_color[3]))
            gc.gui_components['legend']['buttons'][k].override_color(
                        Gtk.StateFlags.NORMAL,
                        Gdk.RGBA(but_font_color[0], but_font_color[1], but_font_color[2], but_font_color[3]))
            gc.gui_components['legend']['buttons'][k].override_background_color(
                        Gtk.StateFlags.NORMAL,
                        Gdk.RGBA(but_color[0], but_color[1], but_color[2], but_color[3]))
            gc.gui_components['legend']['buttons'][k].connect('clicked', self.toggle_plot_item, gc, k)
            gc.gui_components['legend']['buttons'][k].set_property('width-request', 90)
            gc.gui_components['legend']['buttons'][k].set_property('margin-top', 1)
            gc.gui_components['legend']['buttons'][k].set_property('margin-bottom', 1)
            gc.gui_components['legend']['buttons'][k].set_property('margin-right', 1)
            gc.gui_components['legend']['buttons'][k].set_property('margin-left', 1)
            lbox.pack_start(gc.gui_components['legend']['buttons'][k], True, True, 0)
        grid.attach(lbox, 1, row, 4, 1)
        row += 1
        main_last_row = row

        # Set up bar plots
        grid_bar = Gtk.Grid()
        grid_bar.override_background_color(Gtk.StateType.NORMAL, Gdk.RGBA(0.7, 0.7, 0.7, 1))
        grid.attach(grid_bar, 1, main_last_row, 1, 1)
        brow = 0
        fig_num = 0
        # plot_top_row = row
        for v in [gc.gui_components['sclk_pstate_status'],
                  gc.gui_components['mclk_pstate_status'],
                  gc.gui_components['temp_status']]:
            # Add Bar Plots Titles
            v['title_obj'] = Gtk.Label()
            v['title_obj'].set_markup('<big><b>Card ' + v['df_name'] + '</b>' + '</big>')
            v['title_obj'].override_color(Gtk.StateFlags.NORMAL, Gdk.RGBA(1.0, 1.0, 1.0, 1.0))
            v['title_obj'].set_property('margin-top', 1)
            v['title_obj'].set_property('margin-bottom', 1)
            v['title_obj'].set_property('margin-right', 4)
            v['title_obj'].set_property('margin-left', 4)
            v['title_obj'].set_alignment(0.5, 0.5)
            lbox = Gtk.Box(spacing=box_spacing_val)
            lbox.override_background_color(Gtk.StateType.NORMAL, Gdk.RGBA(.20, .40, .60, 1.0))
            lbox.set_property('margin-top', 1)
            lbox.set_property('margin-bottom', 1)
            lbox.set_property('margin-right', 1)
            lbox.set_property('margin-left', 1)
            lbox.pack_start(v['title_obj'], True, True, 0)

            grid_bar.attach(lbox, 1, brow, 1, 1)
            brow += 1

            # Add Bar Plots
            # Set up plot figure and canvas
            v['figure_num'] = 100 + fig_num
            fig_num += 1
            v['figure'], v['ax1'] = plt.subplots(num=v['figure_num'])
            v['figure'].set_facecolor(gc.colors['figface'])

            plt.figure(v['figure_num'])
            plt.subplots_adjust(left=0.13, right=0.97, top=0.97, bottom=0.1)
            v['ax1'].set_facecolor(gc.colors['plotface'])
            if v['df_name'] == 'temp':
                plt.yticks(np.arange(20, 91, 10))
            else:
                plt.yticks(np.arange(0, 9, 1))

            v['canvas'] = FigureCanvas(v['figure'])  # a Gtk.DrawingArea
            v['canvas'].set_size_request(def_bp_x_size, bp_y_size)

            lbox = Gtk.Box(spacing=box_spacing_val)
            lbox.override_background_color(Gtk.StateType.NORMAL, Gdk.RGBA(0.5, 0.5, 0.5, 1.0))
            lbox.set_property('margin-top', 1)
            lbox.set_property('margin-bottom', 1)
            lbox.set_property('margin-right', 1)
            lbox.set_property('margin-left', 1)
            lbox.pack_start(v['canvas'], True, True, 0)

            grid_bar.attach(lbox, 1, brow, 1, 1)
            brow += 1

        # Set up gpu plots
        grid_plot = Gtk.Grid()
        grid_plot.override_background_color(Gtk.StateType.NORMAL, Gdk.RGBA(0.7, 0.7, 0.7, 1))
        grid.attach(grid_plot, 2, main_last_row, 3, 1)
        prow = 0
        # row = plot_top_row
        for k, v in gc.gui_components['card_plots'].items():
            data_val = ldf[ldf['Card#'].isin([k])]['energy'].iloc[-1]
            model_val = ldf[ldf['Card#'].isin([k])]['model_display'].iloc[-1]
            # Add GPU Plots Titles
            v['title_obj'] = Gtk.Label()
            v['title_obj'].set_markup('<big><b>Card   ' + str(k) + '    ' + str(model_val) +
                                      '    Energy:  ' + str(data_val) + '</b>' + '</big>')
            v['title_obj'].override_color(Gtk.StateFlags.NORMAL, Gdk.RGBA(1.0, 1.0, 1.0, 1.0))
            v['title_obj'].set_property('margin-top', 1)
            v['title_obj'].set_property('margin-bottom', 1)
            v['title_obj'].set_property('margin-right', 4)
            v['title_obj'].set_property('margin-left', 4)
            v['title_obj'].set_alignment(0.5, 0.5)
            lbox = Gtk.Box(spacing=box_spacing_val)
            rgba_col = hex_to_rgba(v['color'])
            lbox.override_background_color(Gtk.StateType.NORMAL,
                                           Gdk.RGBA(rgba_col[0], rgba_col[1], rgba_col[2], rgba_col[3]))
            lbox.set_property('margin-top', 1)
            lbox.set_property('margin-bottom', 1)
            lbox.set_property('margin-right', 1)
            lbox.set_property('margin-left', 1)
            lbox.pack_start(v['title_obj'], True, True, 0)

            grid_plot.attach(lbox, 1, prow, 1, 1)
            prow += 1

            # Add GPU Plots
            # Set up plot figure and canvas
            v['figure_num'] = 500 + k
            v['figure'], v['ax1'] = plt.subplots(num=v['figure_num'])
            v['figure'].set_facecolor(gc.colors['figface'])
            plt.figure(v['figure_num'])
            plt.subplots_adjust(left=0.1, right=0.9, top=0.97, bottom=0.03)

            v['ax1'].set_facecolor(gc.colors['plotface'])
            v['ax1'].set_xticks([])
            v['ax1'].set_xticklabels([])
            v['ax1'].set_yticks(np.arange(0, 250, 20))
            v['ax1'].tick_params(axis='y', which='major', labelsize=8)

            v['ax2'] = v['ax1'].twinx()
            v['ax2'].set_xticks([])
            v['ax2'].set_xticklabels([])
            v['ax2'].set_yticks(np.arange(500, 1500, 100))
            v['ax2'].tick_params(axis='y', which='major', labelsize=8)

            v['canvas'] = FigureCanvas(v['figure'])  # a Gtk.DrawingArea
            v['canvas'].set_size_request(def_gp_x_size, gp_y_size)

            lbox = Gtk.Box(spacing=box_spacing_val)
            lbox.override_background_color(Gtk.StateType.NORMAL, Gdk.RGBA(1, 1, 1, 1.0))
            lbox.set_property('margin-top', 1)
            lbox.set_property('margin-bottom', 1)
            lbox.set_property('margin-right', 1)
            lbox.set_property('margin-left', 1)
            lbox.pack_start(v['canvas'], True, True, 0)

            grid_plot.attach(lbox, 1, prow, 1, 1)
            prow += 1

    def toggle_plot_item(self, parent, gc, k):
        if gc.plot_items[k]:
            gc.plot_items[k] = False
        else:
            gc.plot_items[k] = True


def updateData(gc, plot_data):
    # SEMAPHORE ###########
    pd_sem.acquire()
    #######################
    ldf = plot_data.df
    try:
        time_val = ldf[ldf['Card#'].isin([plot_data.gpu_list[0]])]['Time'].iloc[-1]
        gc.gui_components['info_bar']['gtk_obj'].set_markup('<big><b>Time   ' + str(time_val) + '</b></big>')
        # Update Bar Plots
        for v in [gc.gui_components['sclk_pstate_status'],
                  gc.gui_components['mclk_pstate_status'],
                  gc.gui_components['temp_status']]:
            data_val = []
            label_val = []
            bar_col = []
            # Set Plot Parameters
            for k in plot_data.gpu_list:
                l, d = ldf[ldf['Card#'].isin([k])][['Card#', v['df_name']]].iloc[-1]
                label_val.append(int(l))
                data_val.append(float(d))
                bar_col.append(gc.gpu_color[l])
            ind = np.arange(gc.num_gpus)  # the x locations for the groups
            width = 0.65       # the width of the bars
    
            # Do bar plot
            plt.figure(v['figure_num'])
            v['ax1'].clear()
            rects1 = v['ax1'].bar(ind, data_val, width, color=bar_col, tick_label=label_val)
            if v['df_name'] == 'temp':
                for a, b in zip(ind, data_val):
                    v['ax1'].text(x=float(a)-(float(width)/1.8), y=0.90*b, s=' '+str(b), fontsize=8)
                plt.ylim((20, 91))
                #v['ax1'].set_ylabel('Temp (C)', color='k')
            else:
                data_val = list(map(int, data_val))
                for a, b in zip(ind, data_val):
                    if b == 0:
                        y_val = b + width
                    else:
                        y_val = b - width
                    v['ax1'].text(x=a-width/4.0, y=y_val, s=str(b), fontsize=10)
                plt.ylim((0, 8))
            v['canvas'].draw()
            v['canvas'].flush_events()

        # Update GPU Plots
        y1lim_max_val = 10*(ldf.loc[:, ['loading', 'power_cap', 'power', 'temp']].max().max() // 10) + 5
        y1lim_min_val = 10*(ldf.loc[:, ['loading', 'power_cap', 'power', 'temp']].min().min() // 10) - 5
        y2lim_max_val = 100*(ldf.loc[:, ['vddgfx', 'sclk_f', 'mclk_f']].max().max() // 100) + 300
        y2lim_min_val = 100*(ldf.loc[:, ['vddgfx', 'sclk_f', 'mclk_f']].min().min() // 100) - 100
        for k, v in gc.gui_components['card_plots'].items():
            data_val = ldf[ldf['Card#'].isin([k])]['energy'].iloc[-1]
            model_val = ldf[ldf['Card#'].isin([k])]['model_display'].iloc[-1]
            v['title_obj'].set_markup('<big><b>Card   ' + str(k) + '    ' + str(model_val) +
                                      '    Energy:  ' + str(data_val) + '</b>' + '</big>')
    
            # Plot GPUs
            plt.figure(v['figure_num'])
            v['ax1'].set_xticklabels([])
            v['ax1'].clear()
            v['ax1'].set_ylabel('Loading/Power/Temp', color='k', fontsize=10)
            for plot_item in ['loading', 'power_cap', 'power', 'temp']:
                if gc.plot_items[plot_item]:
                    v['ax1'].plot(ldf[ldf['Card#'].isin([k])]['datetime'],
                                  ldf[ldf['Card#'].isin([k])][plot_item],
                                  color=gc.colors[plot_item], linewidth=0.5)
                    v['ax1'].text(x=ldf[ldf['Card#'].isin([k])]['datetime'].iloc[-1],
                                  y=ldf[ldf['Card#'].isin([k])][plot_item].iloc[-1],
                                  s=str(int(ldf[ldf['Card#'].isin([k])][plot_item].iloc[-1])),
                                  bbox=dict(boxstyle='round,pad=0.2', facecolor=gc.colors[plot_item]), fontsize=6)
    
            v['ax2'].clear()
            v['ax2'].set_xticklabels([])
            v['ax2'].set_ylabel('MHz/mV', color='k', fontsize=10)
            for plot_item in ['vddgfx', 'sclk_f', 'mclk_f']:
                if gc.plot_items[plot_item]:
                    v['ax2'].plot(ldf[ldf['Card#'].isin([k])]['datetime'],
                                  ldf[ldf['Card#'].isin([k])][plot_item],
                                  color=gc.colors[plot_item], linewidth=0.5)
                    v['ax2'].text(x=ldf[ldf['Card#'].isin([k])]['datetime'].iloc[-1],
                                  y=ldf[ldf['Card#'].isin([k])][plot_item].iloc[-1],
                                  s=str(int(ldf[ldf['Card#'].isin([k])][plot_item].iloc[-1])),
                                  bbox=dict(boxstyle='round,pad=0.2', facecolor=gc.colors[plot_item]), fontsize=6)

            v['ax1'].set_yticks(np.arange(y1lim_min_val, y1lim_max_val, 10))
            v['ax2'].set_yticks(np.arange(y2lim_min_val, y2lim_max_val, 100))

            v['canvas'].draw()
            v['canvas'].flush_events()
    except:
        print('matplotlib error, stack size is %d' % get_stack_size())
        plot_data.kill_thread()

    # SEMAPHORE ###########
    pd_sem.release()
    #######################


def read_from_stdin(refreshtime, plot_data):
    # this should continuously from from stdin and populate df and call plot/gui update
    first_update = True
    header = True
    sync_add = 0
    while not plot_data.quit:
        if env.gut_const.SIMLOG: time.sleep(refreshtime/4.0)
        ndf = pd.DataFrame()

        # Process a set of GPUs at a time
        skip_update = False
        read_time = 0.0
        for gpu_index in range(0, plot_data.num_gpus + sync_add):
            tb = env.gut_const.now(env.gut_const.USELTZ)
            line = sys.stdin.readline()
            tmp_read_time = (env.gut_const.now(env.gut_const.USELTZ) - tb).total_seconds()
            if tmp_read_time > read_time:
                read_time = tmp_read_time

            if line == '':
                if env.gut_const.DEBUG: print('Error: Null input line')
                plot_data.kill_thread()
                break
            if header:
                header_item = list(line.strip().split('|'))
                header = False
                continue
            line = line.strip()
            line_item = list(line.strip().split('|'))
            new_line_item = []
            for l in line_item:
                ll = l.strip()
                if ll.isnumeric():
                    new_line_item.append(int(ll))
                elif re.fullmatch(r'[0-9]+.[0-9]*', ll) or re.fullmatch(r'[0-9]*.[0-9]+', ll):
                    new_line_item.append(float(ll))
                elif ll == '' or ll == '-1' or ll == 'NA' or ll is None:
                    new_line_item.append(np.nan)
                else:
                    new_line_item.append(ll)
            line_item = tuple(new_line_item)
            rdf = pd.DataFrame.from_records([line_item], columns=header_item)
            rdf['datetime'] = pd.to_datetime(rdf['Time'])
            ndf = pd.concat([ndf, rdf], ignore_index=True)
            del rdf
            if ndf['Time'].tail(plot_data.num_gpus).nunique() > 1:
                sync_add = 1
            else:
                sync_add = 0

        if env.gut_const.DEBUG:
            print(env.gut_const.now(env.gut_const.USELTZ).strftime('%c'))
            print(ndf)

        if not env.gut_const.SIMLOG:
            if read_time < 0.003:
                skip_update = True
                if env.gut_const.DEBUG: print('skipping update')

        # SEMAPHORE ############
        pd_sem.acquire()
        ########################
        # Concatenate new data on plot_data dataframe and truncate
        plot_data.df = pd.concat([plot_data.df, ndf], ignore_index=True)
        plot_data.df.reset_index(drop=True, inplace=True)

        # Truncate df in place
        plot_length = int(len(plot_data.df.index) / plot_data.num_gpus)
        #print('df length: {}, plot_data.length: {}'.format(plot_length, plot_data.length))
        if plot_length > plot_data.length:
            trun_index = plot_length - plot_data.length
            plot_data.df.drop(np.arange(0, trun_index), inplace=True)
            plot_data.df.reset_index(drop=True, inplace=True)
        # SEMAPHORE ############
        pd_sem.release()
        ########################
        del ndf

        #########################
        # Update plots
        #########################
        if skip_update:
            continue
        if plot_data.gui_comp is None:
            continue
        if plot_data.gui_comp.is_ready():
            if first_update:
                time.sleep(refreshtime)
                first_update = False
            GLib.idle_add(updateData, plot_data.gui_comp, plot_data)
            while Gtk.events_pending():
                Gtk.main_iteration_do(True)
            # SEMAPHORE ############
            time.sleep(0.01)
            pd_sem.acquire()
            pd_sem.release()
            ########################
            garbcollect.collect()
        if env.gut_const.DEBUG: print('update stack size: ', get_stack_size())

    # Quit
    print('exit stack size: ', get_stack_size())
    sys.exit(0)


def read_from_gpus(refreshtime, plot_data):
    # This should continuously from from gpus and populate df and call plot/gui update
    first_update = True
    while not plot_data.quit:
        ndf = pd.DataFrame()

        plot_data.com_gpu_list.read_gpu_sensor_data()
        plot_data.com_gpu_list.read_gpu_state_data()

        # Process a set of GPUs at a time
        skip_update = False
        for k, v in plot_data.com_gpu_list.list.items():
            gpu_plot_data = v.get_plot_data(plot_data.com_gpu_list)
            if env.gut_const.DEBUG: print('gpu_plot_data: ', gpu_plot_data)

            rdf = pd.DataFrame.from_records([tuple(gpu_plot_data.values())], columns=tuple(gpu_plot_data.keys()))
            rdf['datetime'] = pd.to_datetime(rdf['Time'])
            ndf = pd.concat([ndf, rdf], ignore_index=True)
            del rdf

        # SEMAPHORE ############
        pd_sem.acquire()
        ########################
        # Concatenate new data on plot_data dataframe and truncate
        plot_data.df = pd.concat([plot_data.df, ndf], ignore_index=True)
        plot_data.df.reset_index(drop=True, inplace=True)

        # Truncate df in place
        plot_length = int(len(plot_data.df.index) / plot_data.num_gpus)
        #print('df length: {}, plot_data.length: {}'.format(plot_length, plot_data.length))
        if plot_length > plot_data.length:
            trun_index = plot_length - plot_data.length
            plot_data.df.drop(np.arange(0, trun_index), inplace=True)
            plot_data.df.reset_index(drop=True, inplace=True)
        # SEMAPHORE ############
        pd_sem.release()
        ########################
        del ndf

        #########################
        # Update plots
        #########################
        if skip_update:
            continue
        if plot_data.gui_comp is None:
            time.sleep(refreshtime)
            continue
        if plot_data.gui_comp.is_ready():
            if first_update:
                time.sleep(refreshtime)
                first_update = False
            GLib.idle_add(updateData, plot_data.gui_comp, plot_data)
            while Gtk.events_pending():
                Gtk.main_iteration_do(True)
            # SEMAPHORE ############
            time.sleep(0.01)
            pd_sem.acquire()
            pd_sem.release()
            ########################
            garbcollect.collect()
        if env.gut_const.DEBUG: print('update stack size: ', get_stack_size())
        time.sleep(refreshtime)

    # Quit
    print('exit stack size: ', get_stack_size())
    sys.exit(0)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--about', help='README', action='store_true', default=False)
    parser.add_argument('--stdin', help='Read from stdin', action='store_true', default=False)
    parser.add_argument('--simlog', help='Simulate with piped log file', action='store_true', default=False)
    parser.add_argument('--ltz', help='Use local time zone instead of UTC', action='store_true', default=False)
    parser.add_argument('--sleep', help='Number of seconds to sleep between updates', type=int, default=3)
    parser.add_argument('-d', '--debug', help='Debug output', action='store_true', default=False)
    args = parser.parse_args()

    # About me
    if args.about:
        print(__doc__)
        print('Author: ', __author__)
        print('Copyright: ', __copyright__)
        print('Credits: ', __credits__)
        print('License: ', __license__)
        print('Version: ', __version__)
        print('Maintainer: ', __maintainer__)
        print('Status: ', __status__)
        import matplotlib
        print('matplotlib version: ', matplotlib.__version__)
        print('pandas version: ', pd.__version__)
        print('numpy version: ', np.__version__)
        sys.exit(0)

    env.gut_const.PATH = os.path.dirname(str(Path(__file__).resolve()))
    env.gut_const.DEBUG = args.debug
    env.gut_const.SIMLOG = args.simlog
    if args.ltz:
        env.gut_const.USELTZ = True

    if env.gut_const.check_env() < 0:
        print('Error in environment. Exiting...')
        sys.exit(-1)

    # Define graph gui and data components
    plot_data = PlotData()

    if not args.stdin:
        # Check value of AMD Feature mask
        try:
            featuremask = env.gut_const.read_amdfeaturemask()
        except FileNotFoundError:
            print('Cannot read ppfeaturemask. Exiting...')
            sys.exit(-1)
        if featuremask == int(0xffff7fff) or featuremask == int(0xffffffff) or featuremask == int(0xfffd7fff):
            print('AMD Wattman features enabled: %s' % hex(featuremask))
        else:
            print('AMD Wattman features not enabled: %s, See README file.' % hex(featuremask))
            sys.exit(-1)
    
        env.gut_const.get_amd_driver_version()
    
        # Get list of AMD GPUs and get basic non-driver details
        gpu_list = GPU.GPU_LIST()
        gpu_list.get_gpu_list()
        gpu_list.read_allgpu_pci_info()
    
        # Check list of AMD GPUs
        num_amd_gpus = gpu_list.num_gpus()
        num_com_gpus = gpu_list.num_compatible_gpus()
        if num_amd_gpus == 0:
            print('No AMD GPUs detected, exiting...')
            sys.exit(-1)
        else:
            if num_com_gpus == 0:
                print('None are compatible, exiting...')
                sys.exit(-1)
            print(f'{num_amd_gpus} AMD GPUs detected, {num_com_gpus} may be compatible, checking...')


        # Read data static driver information for GPUs
        gpu_list.read_gpu_driver_info()
        gpu_list.read_gpu_sensor_static_data()
        # Read dynamic sensor and state data from GPUs
        gpu_list.read_gpu_sensor_data()
        gpu_list.read_gpu_state_data()
    
        # Check number of compatible GPUs again
        num_com_gpus = gpu_list.num_compatible_gpus()
        if num_com_gpus == 0:
            print('None are compatible, exiting...')
            sys.exit(-1)
        else:
            print(f'{num_com_gpus} are confirmed compatible.')
            print('')
    
        # Generate a new list of only compatible GPUs
        plot_data.com_gpu_list = gpu_list.list_compatible_gpus()
        plot_data.num_gpus = num_com_gpus
    # end of if args.stdin == False

    if args.stdin or args.simlog:
        readthread = threading.Thread(target=read_from_stdin, daemon=True, args=[args.sleep, plot_data]).start()
    else:
        readthread = threading.Thread(target=read_from_gpus, daemon=True, args=[args.sleep, plot_data]).start()

    print('%s waiting for initial data' % __program_name__, end='', flush=True)
    while len(plot_data.df.index) < 9:
        print('.', end='', flush=True)
        time.sleep(args.sleep/4.0)
    print('')

    # After reading initial data, set gpus
    plot_data.set_gpus()

    gc = GuiComponents(plot_data)
    gplot = GPUPlotWindow(gc, plot_data)
    gplot.connect('delete-event', Gtk.main_quit)
    gplot.show_all()
    gc.set_ready(True)
    Gtk.main()
    plot_data.kill_thread()


if __name__ == '__main__':
    main()
