/*
    TCFbEDataSetEx.cpp    June 6, 2004.

    Copyright (C) 2003-2004 CFbE Research Group,
    Software Engineering Laboratory,
    Graduate School of Information Science,
    Nara Institute of Science and Technology,
    All rights reserved.

    This program is free software; you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2, (at your option) or
    any later version.

    This program is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with GNU Emacs; see the file COPYING.  If not, write to the
    Free Software Foundation, Inc., 59 Temple Place - Suite 330,
    Boston, MA 02111-1307, USA.
*/
//---------------------------------------------------------------------------
#pragma hdrstop

#include "TCFbEDataSetEx.h"
//---------------------------------------------------------------------------
#pragma package(smart_init)            
//---------------------------------------------------------------------------
double __fastcall TCFbEDataSetEx::GetSimilarityByIndex(int TargetRowIndex, int ComparedRowIndex)
{
    if (!this->IsPrepared()) {
        Exception("TCFbEDataSetEx is not preapared (in TCFbEDataSetEx::GetSimilaritiesByIndex).");
    }

    if (!this->SimilarityCache->GetEnabledByIndex(TargetRowIndex, ComparedRowIndex)) {
        double CalculatedSimilarity = this->FSimilarityComputationAlgorithm->ComputeSimilarity(this->SimilarityComputationTable, TargetRowIndex, ComparedRowIndex);

        if (this->FCaseAmplifier != 1.0) {
            if (CalculatedSimilarity >= 0.0) {
                CalculatedSimilarity = Power(CalculatedSimilarity, this->FCaseAmplifier);
            } else {
                CalculatedSimilarity = -1.0 * Power(-1.0 * CalculatedSimilarity, this->FCaseAmplifier);
            }
        }

        this->SimilarityCache->SetValueByIndex(TargetRowIndex, ComparedRowIndex, CalculatedSimilarity);
    }

    return this->SimilarityCache->GetValueByIndex(TargetRowIndex, ComparedRowIndex);
}

//---------------------------------------------------------------------------
double __fastcall TCFbEDataSetEx::GetPredictedValueByIndex(int TargetRowIndex, int TargetColumnIndex)
{
    if (!this->IsPrepared()) {
        Exception("TCFbEDataSetEx is not preapared (in TCFbEDataSetEx::GetPredictedValueByIndex).");
    }

    if (!this->PredictedValueCache->GetEnabledByIndex(TargetRowIndex, TargetColumnIndex)) {
        double  PredictedValue;
        TObjectList* NeighborList = new TObjectList();
        if (this->FAlgorithmBase == abRow) {
            this->GetNearestNeighbors(TargetRowIndex, TargetColumnIndex, this->FNeighborhoodSize, NeighborList);
            PredictedValue = this->FPredictionAlgorithm->Predict(this->PredictionTable, TargetRowIndex, TargetColumnIndex, NeighborList);
        } else {    // アイテムベースなら予測対象の行と列を入替えて実行
            this->GetNearestNeighbors(TargetColumnIndex, TargetRowIndex, this->FNeighborhoodSize, NeighborList);
            PredictedValue = this->FPredictionAlgorithm->Predict(this->PredictionTable, TargetColumnIndex, TargetRowIndex, NeighborList);
        }
        delete NeighborList;

        switch (this->FValueBase) {
            case vbStandardizedValue:
                PredictedValue = this->DisStandardize(PredictedValue, this->GetAverageOfColumn(TargetColumnIndex), this->GetStdevOfColumn(TargetColumnIndex));
                break;
            case vbNormalizedValue:
                PredictedValue = this->DisNormalize(PredictedValue, this->GetMaxOfColumn(TargetColumnIndex), this->GetMinOfColumn(TargetColumnIndex));
                break;
            case vbOrder:
                PredictedValue = this->DisOrderingEx(PredictedValue, TargetColumnIndex);
                break;
            default:
                ; // 何もしない
        }
        this->PredictedValueCache->SetValueByIndex(TargetRowIndex, TargetColumnIndex, PredictedValue);
    }

    return this->PredictedValueCache->GetValueByIndex(TargetRowIndex, TargetColumnIndex);
}

//---------------------------------------------------------------------------
// コンストラクタ
__fastcall TCFbEDataSetEx::TCFbEDataSetEx(void)
    : TCFbETableEx()
{
    this->FValueBase = vbUnknown;
    this->FAlgorithmBase = abUnknown;
    this->FSimilarityComputationAlgorithm = NULL;
    this->FPredictionAlgorithm = NULL;
    this->FNeighborhoodSize = -1;
    this->FIsPrepared = false;

    this->TemporallyUnableRowIndex = -1;
    this->TemporallyUnableColumnIndex = -1;
    this->TemporallyUnableValue = 0.0;
}

//---------------------------------------------------------------------------
// コンストラクタ
// SourceCsvStringList からデータを構築する
// Row の 1 行目にレコードのラベルが
// Column の 1 行目にアイテムのラベルが
// 後は値が入っているとみなす
__fastcall TCFbEDataSetEx::TCFbEDataSetEx(TStringList* SourceCsvStringList)
    : TCFbETableEx(SourceCsvStringList)
{
    this->FValueBase = vbUnknown;
    this->FAlgorithmBase = abUnknown;
    this->FSimilarityComputationAlgorithm = NULL;
    this->FPredictionAlgorithm = NULL;
    this->FNeighborhoodSize = -1;
    this->FIsPrepared = false;

    this->TemporallyUnableRowIndex = -1;
    this->TemporallyUnableColumnIndex = -1;
    this->TemporallyUnableValue = 0.0;
}

//---------------------------------------------------------------------------
// デストラクタ
__fastcall TCFbEDataSetEx::~TCFbEDataSetEx()
{
    if (this->IsPrepared()) {
        delete this->FPredictionAlgorithm;
        delete this->FSimilarityComputationAlgorithm;
    }
}

//---------------------------------------------------------------------------
// キャッシュを初期化する
void __fastcall TCFbEDataSetEx::InitCache(void)
{
    this->NormalizedTable = new TCFbETableEx();
    this->AddChildCache(this->NormalizedTable, "NormalizedTable", false);
    
    TCFbETableEx::InitCache();
}

//---------------------------------------------------------------------------
//    CF の準備を行う（PredictionTable を準備する
//    ValueBase:            計算に使う値（Value / StandardizedValue / NormalizedValue / Order
//    AlgorithmBase:        Row(User)-based か，Column(Item)-based か
//    SimilarityComputationAlgorithm:   類似度計算アルゴリズム（TCFbEDataSetEx が解放する
//    PredictionAlgorithm:  予測アルゴリズム（TCFbEDataSetEx が解放する
//    NeighborhoodSize:     ネイバーフッドサイズ
//    DefaultValueEnabled:  Default Value 埋め込み機能を有効にするかどうか
//    DefaultValue:         Default Value 埋め込み機能が有効なときに埋め込む値
//    InverseCaseFrequencyEnabled:  Inverse Case Frequency 機能を有効にするかどうか
//    CaseAmplifier:        Case Amplification 機能を使用するときの amplifier（使用しないときは 1.0 を指定）
void __fastcall TCFbEDataSetEx::PrepareCF(TCFbEValueBase ValueBase, TCFbEAlgorithmBase AlgorithmBase, TCFbESimilarityComputationAlgorithm* SimilarityComputationAlgorithm, TCFbEPredictionAlgorithm* PredictionAlgorithm, int NeighborhoodSize, bool DefaultValueEnabled, double DefaultValue, bool InverseCaseFrequencyEnabled, double CaseAmplifier)
{
    if (this->IsPrepared()) {
        delete this->FSimilarityComputationAlgorithm;
        delete this->FPredictionAlgorithm;
        this->NormalizedTable->DeleteChildCache("SimilarityComputationTable");
        this->SimilarityComputationTable = NULL;
        this->NormalizedTable->DeleteChildCache("PredictionTable");
        this->PredictionTable = NULL;

        for (int i = 1; i < this->ChildCacheList->Count; i++) {
            this->DeleteChildCacheByIndex(i);
        }
    }

    this->FSimilarityComputationAlgorithm = SimilarityComputationAlgorithm;
    this->FPredictionAlgorithm = PredictionAlgorithm;
    this->FNeighborhoodSize = NeighborhoodSize;
    this->FAlgorithmBase = AlgorithmBase;
    this->FValueBase = ValueBase;

    // Value / StandardizedValue / NormalizedValue / Order の準備
    this->NormalizedTable->Assign(this);

    switch (this->FValueBase) {
        case vbStandardizedValue:
            this->AddChildCache(new TCFbEColumnCachedTable(this), "Average.column", true);
            this->AddChildCache(new TCFbEColumnCachedTable(this), "Stdev.column", true);
            this->Standardize(NormalizedTable);
            break;

        case vbNormalizedValue:
            this->AddChildCache(new TCFbEColumnCachedTable(this), "Max.column", true);
            this->AddChildCache(new TCFbEColumnCachedTable(this), "Min.column", true);
            this->Normalize(NormalizedTable);
            break;

        case vbOrder :
            this->Ordering(NormalizedTable);
            break;

        default:
            break;
    }

    // Default Value の設定
    this->FDefaultValueEnabled = DefaultValueEnabled;
    this->FDefaultValue = DefaultValue;
    if (this->FDefaultValueEnabled) {
        for (int i = 0; i < this->NormalizedTable->GetNumberOfRows(); i++) {
            for (int j = 0; j < this->NormalizedTable->GetNumberOfColumns(); j++) {
                if (!this->NormalizedTable->GetEnabledByIndex(i, j)) {
                    this->NormalizedTable->SetValueByIndex(i, j, this->FDefaultValue);
                }
            }
        }
    }

    // SimilarityComputationTable, PredictionTable の準備
    if (this->FAlgorithmBase == abRow) {    // User-based CF
        this->SimilarityComputationTable = new TCFbETableEx();
        this->PredictionTable = new TCFbETableEx();

        this->SimilarityComputationTable->CopyAllElementsAndCache(this->NormalizedTable, false, false);
        this->PredictionTable->CopyAllElementsAndCache(this->NormalizedTable, false, false);

        this->NormalizedTable->AddChildCache(this->SimilarityComputationTable, "SimilarityComputationTable", false);
        this->NormalizedTable->AddChildCache(this->PredictionTable, "PredictionTable", false);
    } else {    // Item-based CF
        this->SimilarityComputationTable = new TCFbERotatedTableEx();
        this->PredictionTable = new TCFbERotatedTableEx();

        this->SimilarityComputationTable->RotatedCopyAllElementsAndCache(this->NormalizedTable, false, false);
        this->PredictionTable->RotatedCopyAllElementsAndCache(this->NormalizedTable, false, false);

        this->NormalizedTable->AddChildCache(this->SimilarityComputationTable, "SimilarityComputationTable", false);
        this->NormalizedTable->AddChildCache(this->PredictionTable, "PredictionTable", false);
    }

    // Inverse Case Frequency
    this->FInverseCaseFrequencyEnabled = InverseCaseFrequencyEnabled;
    if (this->FInverseCaseFrequencyEnabled) {
        for (int j = 0; j < this->SimilarityComputationTable->GetNumberOfColumns(); j++) {
            double    ICFCoefficient = this->SimilarityComputationTable->GetICFCoefficient(j);
            for (int i = 0; i < this->SimilarityComputationTable->GetNumberOfRows(); i++) {
                if (this->SimilarityComputationTable->GetEnabledByIndex(i, j)) {
                    this->SimilarityComputationTable->SetValueByIndex(i, j, this->SimilarityComputationTable->GetValueByIndex(i, j) * ICFCoefficient);
                }
            }
        }
    }

    this->FCaseAmplifier = CaseAmplifier;

    // Similarity キャッシュの初期化
    this->SimilarityCache = new TCFbERowRowCachedTable(this->SimilarityComputationTable->GetRowLabelList(), this->SimilarityComputationTable->GetColumnLabelList());
    this->SimilarityComputationTable->AddChildCache(this->SimilarityCache, "SimilarityCache", true);

    // PredictedValue キャッシュの初期化
    if (this->FAlgorithmBase == abRow) {    // User-based CF
        this->PredictedValueCache = new TCFbETableEx(this->GetRowLabelList(), this->GetColumnLabelList());
    } else {    // Item-based CF
        this->PredictedValueCache = new TCFbERotatedTableEx(this->GetRowLabelList(), this->GetColumnLabelList());
    }
    this->PredictionTable->AddChildCache(this->PredictedValueCache, "PredictedValueCache", true);

    if (this->FSimilarityComputationAlgorithm != NULL) {
        this->FSimilarityComputationAlgorithm->PrepareCache(this->SimilarityComputationTable);
    }
    if (this->FPredictionAlgorithm != NULL) {
        this->FPredictionAlgorithm->PrepareCache(this->PredictionTable);
    }

    this->FIsPrepared = true;
}

//---------------------------------------------------------------------------
// TargetRowIndex に対する NearestNeigobors を調べる
// TargetRowIndex: 類似度を調べる対象の行
// TargetColumnIndex: 予測対象の列（TargetColumnIndex の値が Enabled でない行は Neighbors に含めない 
// NeighborList に Neigobors を類似度が高い順に入れて返す
// 返された TObjectList は GetNearestNeighbors を呼んだクラスが削除する必要がある
void __fastcall TCFbEDataSetEx::GetNearestNeighbors(int TargetRowIndex, int TargetColumnIndex, int NeighborhoodSize, TObjectList* NeighborList)
{
    if (NeighborhoodSize == 0) {
        NeighborhoodSize = MaxInt;
    }

    for (int i = 0; i < this->SimilarityComputationTable->GetNumberOfRows(); i++) {
        if (this->PredictionTable->GetEnabledByIndex(i, TargetColumnIndex) && (i != TargetRowIndex)) {
            double Similarity = this->GetSimilarityByIndex(TargetRowIndex, i);

            int k = 0;
            while (k < NeighborList->Count) {
                if ( ((TCFbENeighbor*)NeighborList->Items[k])->GetSimilarity() < Similarity) {
                    break;
                }
                k++;
            }

            if (k < NeighborhoodSize) {
                if (NeighborList->Count >= NeighborhoodSize) {
                    NeighborList->Delete(NeighborList->Count - 1);
                }
                TCFbECell* CurrentSourceCell = (this->FAlgorithmBase == abRow) ? this->GetCellByIndex(i, TargetColumnIndex) : this->GetCellByIndex(TargetColumnIndex, i);
                double Value = CurrentSourceCell->GetEnabled() ? CurrentSourceCell->GetValue() : this->FDefaultValue;
                NeighborList->Insert(k, new TCFbENeighbor(this->PredictionTable->GetRowLabelList()->Strings[i], Similarity, this->PredictionTable->GetValueByIndex(i, TargetColumnIndex), Value));
            }
        }
    }
}

//---------------------------------------------------------------------------
// BaseDataSet にあって，this にない列を新たな列として追加する
void __fastcall TCFbEDataSetEx::FillColumns(TCFbEDataSetEx* BaseDataSet)
{
    for (int j = 0; j < BaseDataSet->GetNumberOfColumns(); j++) {
        if (this->GetColumnLabelList()->IndexOf(BaseDataSet->GetColumnLabelList()->Strings[j]) == -1) {
            this->AddColumn(BaseDataSet->GetColumnLabelList()->Strings[j].c_str());
        }
    }
}

//---------------------------------------------------------------------------
// TargetDataSet と首尾一貫性があるかどうかを調べる
bool __fastcall TCFbEDataSetEx::IsConsistentWith(TCFbEDataSetEx* TargetDataSet)
{
    if (this->GetNumberOfColumns() != TargetDataSet->GetNumberOfColumns()) {
        return false;
    }

    for (int j = 0; j < this->GetColumnLabelList()->Count; j++) {
        if (TargetDataSet->GetColumnLabelList()->IndexOf(this->GetColumnLabelList()->Strings[j]) == -1) {
            return false;
        }
    }
    return true;
}

//---------------------------------------------------------------------------
// ColumnIndex 列目の Ordering された値 Order を DisOrdering して返す
double __fastcall TCFbEDataSetEx::DisOrderingEx(double Order, int ColumnIndex)
{
    TDoubleDynArray SortedOrders;
    if (this->FAlgorithmBase == abRow) {    // User-based
        this->PredictionTable->GetEnabledCellsInColumn(ColumnIndex, SortedOrders);
    } else {
        this->PredictionTable->GetEnabledCellsInRow(ColumnIndex, SortedOrders);
    }
    TCFbEToolkit::GetToolkit()->SortDoubleDynArray(SortedOrders, sdDescending);

    TDoubleDynArray SortedValues;
    this->GetEnabledCellsInColumn(ColumnIndex, SortedValues);
    TCFbEToolkit::GetToolkit()->SortDoubleDynArray(SortedValues, sdDescending);

    double ResultedValue = this->DisOrdering(Order, SortedOrders, SortedValues);
    SortedOrders.Length = 0;
    SortedValues.Length = 0;
    
    return ResultedValue;
}

//---------------------------------------------------------------------------
// TargetRowIndex, TargetColumnIndex のセルを一時的に Enabled = false にする
void __fastcall TCFbEDataSetEx::TemporallyUnable(int TargetRowIndex, int TargetColumnIndex)
{
    if ((this->TemporallyUnableRowIndex >= 0) && (this->TemporallyUnableColumnIndex >= 0)) {
        this->RecoverEnabled();
    }

    if (this->GetEnabledByIndex(TargetRowIndex, TargetColumnIndex)) {
        this->TemporallyUnableRowIndex = TargetRowIndex;
        this->TemporallyUnableColumnIndex = TargetColumnIndex;
        this->TemporallyUnableValue = this->GetValueByIndex(TargetRowIndex, TargetColumnIndex);
        this->UnableElementByIndex(TargetRowIndex, TargetColumnIndex);
    } else {
        this->TemporallyUnableRowIndex = -1;
        this->TemporallyUnableColumnIndex = -1;
    }
}

//---------------------------------------------------------------------------
// 一時的に Enabled = false にしていたセルを元にもどす
void __fastcall TCFbEDataSetEx::RecoverEnabled(void)
{
    if ((this->TemporallyUnableRowIndex < 0) && (this->TemporallyUnableColumnIndex < 0)) {
        return;
    }

    this->SetValueByIndex(this->TemporallyUnableRowIndex, this->TemporallyUnableColumnIndex, this->TemporallyUnableValue);
    this->TemporallyUnableRowIndex = -1;
    this->TemporallyUnableColumnIndex = -1;
}

//---------------------------------------------------------------------------
// 以下，protected メソッドの定義
//---------------------------------------------------------------------------
void __fastcall TCFbEDataSetEx::RePrepareCF(int TargetRowIndex, int TargetColumnIndex)
{
    if (!this->IsPrepared()) {
        return;
//        Exception("TCFbEDataSetEx is not preapared (in TCFbEDataSetEx::RePrepareCF).");
    }

    // 対象セルが含まれる行・列の正規化やり直し
    // Value / StandardizedValue / NormalizedValue / Order の準備
    switch (this->FValueBase) {
        case vbStandardizedValue:
        case vbNormalizedValue:
        case vbOrder:
            this->NormalizedTable->CopyColumnFrom(this, TargetColumnIndex, true);

            if (this->FValueBase == vbStandardizedValue) {
                this->PartialStandardize(NormalizedTable, TargetColumnIndex);
            } else if (this->FValueBase == vbNormalizedValue) {
                this->PartialNormalize(NormalizedTable, TargetColumnIndex);
            } else if (this->FValueBase == vbOrder) {
                this->PartialOrdering(NormalizedTable, TargetColumnIndex);
            }

            // Default Value の設定
            if (this->FDefaultValueEnabled) {
                for (int i = 0; i < this->NormalizedTable->GetNumberOfRows(); i++) {
                    if (!this->NormalizedTable->GetEnabledByIndex(i, TargetColumnIndex) && (i != TargetRowIndex)) {
                        this->NormalizedTable->SetValueByIndex(i, TargetColumnIndex, this->FDefaultValue);
                    }
                }
            }

            // SimilarityComputationTable, PredictionTable の準備
            if (this->FAlgorithmBase == abRow) {    // User-based CF
                this->SimilarityComputationTable->CopyColumnFrom(this->NormalizedTable, TargetColumnIndex, false);
                this->PredictionTable->CopyColumnFrom(this->NormalizedTable, TargetColumnIndex, false);
            } else {    // Item-based CF
                this->SimilarityComputationTable->CopyColumnAsRowFrom(this->NormalizedTable, TargetColumnIndex, false);
                this->PredictionTable->CopyColumnAsRowFrom(this->NormalizedTable, TargetColumnIndex, false);
            }

            break;

        default:
            if (this->FAlgorithmBase == abRow) {    // User-based CF
                this->SimilarityComputationTable->CopyCellFrom(this, TargetRowIndex, TargetColumnIndex, TargetRowIndex, TargetColumnIndex);
                this->PredictionTable->CopyCellFrom(this, TargetRowIndex, TargetColumnIndex, TargetRowIndex, TargetColumnIndex);
            } else {
                this->SimilarityComputationTable->CopyCellFrom(this, TargetRowIndex, TargetColumnIndex, TargetColumnIndex, TargetRowIndex);
                this->PredictionTable->CopyCellFrom(this, TargetRowIndex, TargetColumnIndex, TargetColumnIndex, TargetRowIndex);
            }
            break;
    }

    // Inverse Case Frequency
    if (this->FInverseCaseFrequencyEnabled) {
        for (int j = 0; j < this->SimilarityComputationTable->GetNumberOfColumns(); j++) {
            double ICFCoefficient = this->SimilarityComputationTable->GetICFCoefficient(j);
            for (int i = 0; i < this->SimilarityComputationTable->GetNumberOfRows(); i++) {
                if (this->SimilarityComputationTable->GetEnabledByIndex(i, j)) {
                    this->SimilarityComputationTable->SetValueByIndex(i, j, this->SimilarityComputationTable->GetValueByIndex(i, j) * ICFCoefficient);
                }
            }
        }
    }
}

//---------------------------------------------------------------------------

