/*
    TCFbEDataSetEx.cpp    June 6, 2004.

    Copyright (C) 2003-2004 CFbE Research Group,
    Software Engineering Laboratory,
    Graduate School of Information Science,
    Nara Institute of Science and Technology,
    All rights reserved.

    This program is free software; you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2, (at your option) or
    any later version.

    This program is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with GNU Emacs; see the file COPYING.  If not, write to the
    Free Software Foundation, Inc., 59 Temple Place - Suite 330,
    Boston, MA 02111-1307, USA.
*/
//---------------------------------------------------------------------------
#pragma hdrstop

#include "TCFbEDataSetEx.h"
//---------------------------------------------------------------------------
#pragma package(smart_init)            
//---------------------------------------------------------------------------
double __fastcall TCFbEDataSetEx::GetSimilarityByIndex(int TargetRowIndex, int ComparedRowIndex)
{
    if (!this->IsPrepared()) {
        Exception("TCFbEDataSetEx is not preapared (in TCFbEDataSetEx::GetSimilaritiesByIndex).");
    }

    TCFbERowRowCachedTable* SimilarityCache = (TCFbERowRowCachedTable*)this->GetSimilarityCache();

    if (!SimilarityCache->GetEnabledByIndex(TargetRowIndex, ComparedRowIndex)) {
        double CalculatedSimilarity = this->FSimilarityComputationAlgorithm->ComputeSimilarity(this->GetSimilarityComputationTable(), TargetRowIndex, ComparedRowIndex);

        if (this->FCaseAmplifier != 1.0) {
            if (CalculatedSimilarity >= 0.0) {
                CalculatedSimilarity = Power(CalculatedSimilarity, this->FCaseAmplifier);
            } else {
                CalculatedSimilarity = -1.0 * Power(-1.0 * CalculatedSimilarity, this->FCaseAmplifier);
            }
        }

        SimilarityCache->SetValueByIndex(TargetRowIndex, ComparedRowIndex, CalculatedSimilarity);
    }

    return SimilarityCache->GetValueByIndex(TargetRowIndex, ComparedRowIndex);
}

//---------------------------------------------------------------------------
double __fastcall TCFbEDataSetEx::GetPredictedValueByIndex(int TargetRowIndex, int TargetColumnIndex)
{
    if (!this->IsPrepared()) {
        Exception("TCFbEDataSetEx is not preapared (in TCFbEDataSetEx::GetPredictedValueByIndex).");
    }

    TCFbECustomTableEx* PredictedValueCache = this->GetPredictedValueCache();

    if (!PredictedValueCache->GetEnabledByIndex(TargetRowIndex, TargetColumnIndex)) {
        double  PredictedValue;
        TObjectList* NeighborList = new TObjectList();
        if (this->FAlgorithmBase == abRow) {
            this->GetNearestNeighbors(TargetRowIndex, TargetColumnIndex, this->FNeighborhoodSize, NeighborList);
            PredictedValue = this->FPredictionAlgorithm->Predict(this->GetPredictionTable(), TargetRowIndex, TargetColumnIndex, NeighborList);
        } else {    // アイテムベースなら予測対象の行と列を入替えて実行
            this->GetNearestNeighbors(TargetColumnIndex, TargetRowIndex, this->FNeighborhoodSize, NeighborList);
            PredictedValue = this->FPredictionAlgorithm->Predict(this->GetPredictionTable(), TargetColumnIndex, TargetRowIndex, NeighborList);
        }
        delete NeighborList;

        switch (this->FValueBase) {
            case vbStandardizedValue:
                PredictedValue = this->DisStandardize(TargetColumnIndex, PredictedValue);
                break;
            case vbNormalizedValue:
                PredictedValue = this->DisNormalize(TargetColumnIndex, PredictedValue);
                break;
            case vbOrder:
                PredictedValue = this->DisOrderingEx(TargetColumnIndex, PredictedValue);
                break;
            default:
                ; // 何もしない
        }
        PredictedValueCache->SetValueByIndex(TargetRowIndex, TargetColumnIndex, PredictedValue);
    }

    return PredictedValueCache->GetValueByIndex(TargetRowIndex, TargetColumnIndex);
}

//---------------------------------------------------------------------------
// コンストラクタ
__fastcall TCFbEDataSetEx::TCFbEDataSetEx(void)
    : TCFbETableEx()
{
    this->FValueBase = vbUnknown;
    this->FAlgorithmBase = abUnknown;
    this->FSimilarityComputationAlgorithm = NULL;
    this->FPredictionAlgorithm = NULL;
    this->FNeighborhoodSize = -1;
    this->FIsPrepared = false;

    this->TemporallyUnableRowIndex = -1;
    this->TemporallyUnableColumnIndex = -1;
    this->TemporallyUnableValue = 0.0;
}

//---------------------------------------------------------------------------
// コンストラクタ
// SourceCsvStringList からデータを構築する
// Row の 1 行目にレコードのラベルが
// Column の 1 行目にアイテムのラベルが
// 後は値が入っているとみなす
__fastcall TCFbEDataSetEx::TCFbEDataSetEx(TStringList* SourceCsvStringList)
    : TCFbETableEx(SourceCsvStringList)
{
    this->FValueBase = vbUnknown;
    this->FAlgorithmBase = abUnknown;
    this->FSimilarityComputationAlgorithm = NULL;
    this->FPredictionAlgorithm = NULL;
    this->FNeighborhoodSize = -1;
    this->FIsPrepared = false;

    this->TemporallyUnableRowIndex = -1;
    this->TemporallyUnableColumnIndex = -1;
    this->TemporallyUnableValue = 0.0;
}

//---------------------------------------------------------------------------
// デストラクタ
__fastcall TCFbEDataSetEx::~TCFbEDataSetEx()
{
    if (this->IsPrepared()) {
        delete this->FPredictionAlgorithm;
        delete this->FSimilarityComputationAlgorithm;
    }
}

//---------------------------------------------------------------------------
// キャッシュを初期化する
void __fastcall TCFbEDataSetEx::InitCache(void)
{
    this->AddChildCache(new TCFbETableEx(), "NormalizedTable", false);
    
    TCFbETableEx::InitCache();
}

//---------------------------------------------------------------------------
//    CF の準備を行う（PredictionTable を準備する
//    ValueBase:            計算に使う値（Value / StandardizedValue / NormalizedValue / Order
//    AlgorithmBase:        Row(User)-based か，Column(Item)-based か
//    SimilarityComputationAlgorithm:   類似度計算アルゴリズム（TCFbEDataSetEx が解放する
//    PredictionAlgorithm:  予測アルゴリズム（TCFbEDataSetEx が解放する
//    NeighborhoodSize:     ネイバーフッドサイズ
//    DefaultValueEnabled:  Default Value 埋め込み機能を有効にするかどうか
//    DefaultValue:         Default Value 埋め込み機能が有効なときに埋め込む値
//    InverseCaseFrequencyEnabled:  Inverse Case Frequency 機能を有効にするかどうか
//    CaseAmplifier:        Case Amplification 機能を使用するときの amplifier（使用しないときは 1.0 を指定）
void __fastcall TCFbEDataSetEx::PrepareCF(TCFbEValueBase ValueBase, TCFbEAlgorithmBase AlgorithmBase, TCFbESimilarityComputationAlgorithm* SimilarityComputationAlgorithm, TCFbEPredictionAlgorithm* PredictionAlgorithm, int NeighborhoodSize, bool DefaultValueEnabled, double DefaultValue, bool InverseCaseFrequencyEnabled, double CaseAmplifier)
{
    TCFbETableEx* NormalizedTable = this->GetNormalizedTable();
    if (this->IsPrepared()) {
        delete this->FSimilarityComputationAlgorithm;
        delete this->FPredictionAlgorithm;
        NormalizedTable->DeleteChildCache("SimilarityComputationTable");
        NormalizedTable->DeleteChildCache("PredictionTable");

        for (int i = 1; i < this->ChildCacheList->Count; i++) {
            this->DeleteChildCacheByIndex(i);
        }
    }

    this->FSimilarityComputationAlgorithm = SimilarityComputationAlgorithm;
    this->FPredictionAlgorithm = PredictionAlgorithm;
    this->FNeighborhoodSize = NeighborhoodSize;
    this->FAlgorithmBase = AlgorithmBase;
    this->FValueBase = ValueBase;

    // Value / StandardizedValue / NormalizedValue / Order の準備
    NormalizedTable->Assign(this);

    switch (this->FValueBase) {
        case vbStandardizedValue:
            this->AddChildCache(new TCFbEColumnCachedTable(this), "Average.column", true);
            this->AddChildCache(new TCFbEColumnCachedTable(this), "Stdev.column", true);
            NormalizedTable->AddChildCache(new TCFbEColumnCachedTable(NormalizedTable), "Average.column", true);
            NormalizedTable->AddChildCache(new TCFbEColumnCachedTable(NormalizedTable), "Stdev.column", true);
            NormalizedTable->Standardize();
            break;

        case vbNormalizedValue:
            this->AddChildCache(new TCFbEColumnCachedTable(this), "Max.column", true);
            this->AddChildCache(new TCFbEColumnCachedTable(this), "Min.column", true);
            NormalizedTable->AddChildCache(new TCFbEColumnCachedTable(NormalizedTable), "Max.column", true);
            NormalizedTable->AddChildCache(new TCFbEColumnCachedTable(NormalizedTable), "Min.column", true);
            NormalizedTable->Normalize();
            break;

        case vbOrder :
            NormalizedTable->Ordering();
            break;

        default:
            break;
    }

    // Default Value の設定
    this->FDefaultValueEnabled = DefaultValueEnabled;
    this->FDefaultValue = DefaultValue;
    if (this->FDefaultValueEnabled) {
        for (int i = 0; i < NormalizedTable->GetNumberOfRows(); i++) {
            for (int j = 0; j < NormalizedTable->GetNumberOfColumns(); j++) {
                if (!NormalizedTable->GetEnabledByIndex(i, j)) {
                    NormalizedTable->SetValueByIndex(i, j, this->FDefaultValue);
                }
            }
        }
    }

    TCFbECustomTableEx* SimilarityComputationTable;
    TCFbECustomTableEx* PredictionTable;

    // SimilarityComputationTable, PredictionTable の準備
    if (this->FAlgorithmBase == abRow) {    // User-based CF
        SimilarityComputationTable = new TCFbETableEx();
        PredictionTable = new TCFbETableEx();

        SimilarityComputationTable->CopyAllElementsAndCache(NormalizedTable, false, false);
        PredictionTable->CopyAllElementsAndCache(NormalizedTable, false, false);

        NormalizedTable->AddChildCache(SimilarityComputationTable, "SimilarityComputationTable", false);
        NormalizedTable->AddChildCache(PredictionTable, "PredictionTable", false);
    } else {    // Item-based CF
        SimilarityComputationTable = new TCFbERotatedTableEx();
        PredictionTable = new TCFbERotatedTableEx();

        SimilarityComputationTable->RotatedCopyAllElementsAndCache(NormalizedTable, false, false);
        PredictionTable->RotatedCopyAllElementsAndCache(NormalizedTable, false, false);

        NormalizedTable->AddChildCache(SimilarityComputationTable, "SimilarityComputationTable", false);
        NormalizedTable->AddChildCache(PredictionTable, "PredictionTable", false);
    }

    // Inverse Case Frequency
    this->FInverseCaseFrequencyEnabled = InverseCaseFrequencyEnabled;
    if (this->FInverseCaseFrequencyEnabled) {
        for (int j = 0; j < SimilarityComputationTable->GetNumberOfColumns(); j++) {
            double    ICFCoefficient = SimilarityComputationTable->GetICFCoefficient(j);
            for (int i = 0; i < SimilarityComputationTable->GetNumberOfRows(); i++) {
                if (SimilarityComputationTable->GetEnabledByIndex(i, j)) {
                    SimilarityComputationTable->SetValueByIndex(i, j, SimilarityComputationTable->GetValueByIndex(i, j) * ICFCoefficient);
                }
            }
        }
    }

    this->FCaseAmplifier = CaseAmplifier;

    // Similarity キャッシュの初期化
    TCFbERowRowCachedTable* SimilarityCache = new TCFbERowRowCachedTable(SimilarityComputationTable->GetRowLabelList(), SimilarityComputationTable->GetColumnLabelList());
    SimilarityComputationTable->AddChildCache(SimilarityCache, "SimilarityCache", true);

    // PredictedValue キャッシュの初期化
    TCFbECustomTableEx* PredictedValueCache;
    if (this->FAlgorithmBase == abRow) {    // User-based CF
        PredictedValueCache = new TCFbETableEx(this->GetRowLabelList(), this->GetColumnLabelList());
    } else {    // Item-based CF
        PredictedValueCache = new TCFbERotatedTableEx(this->GetRowLabelList(), this->GetColumnLabelList());
    }
    PredictionTable->AddChildCache(PredictedValueCache, "PredictedValueCache", true);

    if (this->FSimilarityComputationAlgorithm != NULL) {
        this->FSimilarityComputationAlgorithm->PrepareCache(this->GetSimilarityComputationTable());
    }
    if (this->FPredictionAlgorithm != NULL) {
        this->FPredictionAlgorithm->PrepareCache(this->GetPredictionTable());
    }

    this->FIsPrepared = true;
}

//---------------------------------------------------------------------------
// TargetRowIndex に対する NearestNeigobors を調べる
// TargetRowIndex: 類似度を調べる対象の行
// TargetColumnIndex: 予測対象の列（TargetColumnIndex の値が Enabled でない行は Neighbors に含めない 
// NeighborList に Neigobors を類似度が高い順に入れて返す
// 返された TObjectList は GetNearestNeighbors を呼んだクラスが削除する必要がある
void __fastcall TCFbEDataSetEx::GetNearestNeighbors(int TargetRowIndex, int TargetColumnIndex, int NeighborhoodSize, TObjectList* NeighborList)
{
    if (NeighborhoodSize == 0) {
        NeighborhoodSize = MaxInt;
    }

    for (int i = 0; i < this->GetSimilarityComputationTable()->GetNumberOfRows(); i++) {
        if (this->GetPredictionTable()->GetEnabledByIndex(i, TargetColumnIndex) && (i != TargetRowIndex)) {
            double Similarity = this->GetSimilarityByIndex(TargetRowIndex, i);

            int k = 0;
            while (k < NeighborList->Count) {
                if ( ((TCFbENeighbor*)NeighborList->Items[k])->GetSimilarity() < Similarity) {
                    break;
                }
                k++;
            }

            if (k < NeighborhoodSize) {
                if (NeighborList->Count >= NeighborhoodSize) {
                    NeighborList->Delete(NeighborList->Count - 1);
                }
                TCFbECell* CurrentSourceCell = (this->FAlgorithmBase == abRow) ? this->GetCellByIndex(i, TargetColumnIndex) : this->GetCellByIndex(TargetColumnIndex, i);
                double Value = CurrentSourceCell->GetEnabled() ? CurrentSourceCell->GetValue() : this->FDefaultValue;
                NeighborList->Insert(k, new TCFbENeighbor(this->GetPredictionTable()->GetRowLabelList()->Strings[i], Similarity, this->GetPredictionTable()->GetValueByIndex(i, TargetColumnIndex), Value));
            }
        }
    }
}

//---------------------------------------------------------------------------
// BaseDataSet にあって，this にない列を新たな列として追加する
void __fastcall TCFbEDataSetEx::FillColumns(TCFbEDataSetEx* BaseDataSet)
{
    for (int j = 0; j < BaseDataSet->GetNumberOfColumns(); j++) {
        if (this->GetColumnLabelList()->IndexOf(BaseDataSet->GetColumnLabelList()->Strings[j]) == -1) {
            this->AddColumn(BaseDataSet->GetColumnLabelList()->Strings[j].c_str());
        }
    }
}

//---------------------------------------------------------------------------
// TargetDataSet と首尾一貫性があるかどうかを調べる
bool __fastcall TCFbEDataSetEx::IsConsistentWith(TCFbEDataSetEx* TargetDataSet)
{
    if (this->GetNumberOfColumns() != TargetDataSet->GetNumberOfColumns()) {
        return false;
    }

    for (int j = 0; j < this->GetColumnLabelList()->Count; j++) {
        if (TargetDataSet->GetColumnLabelList()->IndexOf(this->GetColumnLabelList()->Strings[j]) == -1) {
            return false;
        }
    }
    return true;
}

//---------------------------------------------------------------------------
// ColumnIndex 列目の Ordering された値 Order を DisOrdering して返す
double __fastcall TCFbEDataSetEx::DisOrderingEx(int ColumnIndex, double Order)
{
    TList* SortedList = new TList();
    if (this->FAlgorithmBase == abRow) {    // User-based
        this->GetPredictionTable()->GetSortedCellListOfColumn(ColumnIndex, SortedList);
    } else {
        this->GetPredictionTable()->GetSortedCellListOfRow(ColumnIndex, SortedList);
    }

    TDoubleDynArray Array;
    Array.Length = SortedList->Count;
    for (int k = 0; k < SortedList->Count; k++) {
        Array[k] = ((TCFbECell*)SortedList->Items[k])->GetValue();
    }

    double ResultedValue = this->DisOrdering(ColumnIndex, Order, Array);
    delete SortedList;
    Array.Length = 0;
    
    return ResultedValue;
}

//---------------------------------------------------------------------------
// TargetRowIndex, TargetColumnIndex のセルを一時的に Enabled = false にする
void __fastcall TCFbEDataSetEx::TemporallyUnable(int TargetRowIndex, int TargetColumnIndex)
{
    if ((this->TemporallyUnableRowIndex >= 0) && (this->TemporallyUnableColumnIndex >= 0)) {
        this->RecoverEnabled();
    }

    if (this->GetEnabledByIndex(TargetRowIndex, TargetColumnIndex)) {
        this->TemporallyUnableRowIndex = TargetRowIndex;
        this->TemporallyUnableColumnIndex = TargetColumnIndex;
        this->TemporallyUnableValue = this->GetValueByIndex(TargetRowIndex, TargetColumnIndex);
        this->UnableElementByIndex(TargetRowIndex, TargetColumnIndex);
    } else {
        this->TemporallyUnableRowIndex = -1;
        this->TemporallyUnableColumnIndex = -1;
    }
}

//---------------------------------------------------------------------------
// 一時的に Enabled = false にしていたセルを元にもどす
void __fastcall TCFbEDataSetEx::RecoverEnabled(void)
{
    if ((this->TemporallyUnableRowIndex < 0) && (this->TemporallyUnableColumnIndex < 0)) {
        return;
    }

    this->SetValueByIndex(this->TemporallyUnableRowIndex, this->TemporallyUnableColumnIndex, this->TemporallyUnableValue);
    this->TemporallyUnableRowIndex = -1;
    this->TemporallyUnableColumnIndex = -1;
}

//---------------------------------------------------------------------------
// 以下，protected メソッドの定義
//---------------------------------------------------------------------------
void __fastcall TCFbEDataSetEx::RePrepareCF(int TargetRowIndex, int TargetColumnIndex)
{
    if (!this->IsPrepared()) {
        return;
//        Exception("TCFbEDataSetEx is not preapared (in TCFbEDataSetEx::RePrepareCF).");
    }

    // 対象セルが含まれる行・列の正規化やり直し
    TCFbECustomTableEx* SimilarityComputationTable = this->GetSimilarityComputationTable();
    TCFbECustomTableEx* PredictionTable = this->GetPredictionTable();
    TCFbETableEx*       NormalizedTable = this->GetNormalizedTable();

    // Value / StandardizedValue / NormalizedValue / Order の準備
    switch (this->FValueBase) {
        case vbStandardizedValue:
        case vbNormalizedValue:
        case vbOrder:
            NormalizedTable->CopyColumnFrom(this, TargetColumnIndex, true);

            if (this->FValueBase == vbStandardizedValue) {
                NormalizedTable->PartialStandardize(TargetColumnIndex);
            } else if (this->FValueBase == vbNormalizedValue) {
                NormalizedTable->PartialNormalize(TargetColumnIndex);
            } else if (this->FValueBase == vbOrder) {
                NormalizedTable->PartialOrdering(TargetColumnIndex);
            }

            // Default Value の設定
            if (this->FDefaultValueEnabled) {
                for (int i = 0; i < NormalizedTable->GetNumberOfRows(); i++) {
                    if (!NormalizedTable->GetEnabledByIndex(i, TargetColumnIndex) && (i != TargetRowIndex)) {
                        NormalizedTable->SetValueByIndex(i, TargetColumnIndex, this->FDefaultValue);
                    }
                }
            }

            // SimilarityComputationTable, PredictionTable の準備
            if (this->FAlgorithmBase == abRow) {    // User-based CF
                SimilarityComputationTable->CopyColumnFrom(NormalizedTable, TargetColumnIndex, false);
                PredictionTable->CopyColumnFrom(NormalizedTable, TargetColumnIndex, false);
            } else {    // Item-based CF
                SimilarityComputationTable->CopyColumnAsRowFrom(NormalizedTable, TargetColumnIndex, false);
                PredictionTable->CopyColumnAsRowFrom(NormalizedTable, TargetColumnIndex, false);
            }

            break;

        default:
            if (this->FAlgorithmBase == abRow) {    // User-based CF
                SimilarityComputationTable->CopyCellFrom(this, TargetRowIndex, TargetColumnIndex, TargetRowIndex, TargetColumnIndex);
                PredictionTable->CopyCellFrom(this, TargetRowIndex, TargetColumnIndex, TargetRowIndex, TargetColumnIndex);
            } else {
                SimilarityComputationTable->CopyCellFrom(this, TargetRowIndex, TargetColumnIndex, TargetColumnIndex, TargetRowIndex);
                PredictionTable->CopyCellFrom(this, TargetRowIndex, TargetColumnIndex, TargetColumnIndex, TargetRowIndex);
            }
            break;
    }

    // Inverse Case Frequency
    if (this->FInverseCaseFrequencyEnabled) {
        for (int j = 0; j < SimilarityComputationTable->GetNumberOfColumns(); j++) {
            double ICFCoefficient = SimilarityComputationTable->GetICFCoefficient(j);
            for (int i = 0; i < SimilarityComputationTable->GetNumberOfRows(); i++) {
                if (SimilarityComputationTable->GetEnabledByIndex(i, j)) {
                    SimilarityComputationTable->SetValueByIndex(i, j, SimilarityComputationTable->GetValueByIndex(i, j) * ICFCoefficient);
                }
            }
        }
    }
}

//---------------------------------------------------------------------------

