function [mee,ste,mec] = collect (pre, runs, option, dat)
%COLLECT  Collect results from runecoc.
%
%   [M,S,C] = COLLECT('run/b1T200_', 1:100, [5,10], '../data/dat-b1');
%   The 3rd argument is optional for cross-validation.
%   The 4th argument is optional for saving the data.

nseg = 4; % # of outputs per iteration (tr err, te err, tr cost, hyp wgt)
tre = 1; tee = 2; trc = 3; wgt = 4; % offsets of the outputs
if nargin == 0, mee = nseg; ste = [tre tee trc]; return; end % for cmplot.m

if nargin < 3, option = []; end
opt = options([1 1], option); % no cross-validation
gs = opt(1);    % group size in CV
folds = opt(2); % CV folds

% errors at the end of training
ee = []; % training error, test error, training cost, # ite, total time
% error curves...
ec = zeros(0,1);
max_l = 0; last = 0;
% # of method
nmethod = 0; tr_n = 0;
for i = runs
    ei = xload([pre int2str(i)]);
    if nmethod == 0, nmethod = length(ei); end
    % check whether ei is complete
    if (length(ei) < nmethod || length(ei{end}) < last*.9)
        disp(['incomplete results at run# ' int2str(i)]);
        runs = runs(1:find(runs==i)-1);
        break;
    end

    for j = 1:nmethod
        l = length(ei{j}) - 4;
        assert(mod(l,nseg) == 0);
        if tr_n == 0
            tr_n = ei{j}(l+1); te_n = ei{j}(l+2); K = ei{j}(l+3);
        else
            assert(all([tr_n te_n K] == ei{j}(l+1:l+3)));
        end
        ite = l / nseg;
        ei{j}(:,tre:nseg:l) = ei{j}(:,tre:nseg:l) / tr_n * 100;
        ei{j}(:,tee:nseg:l) = ei{j}(:,tee:nseg:l) / te_n * 100;
        ei{j}(:,trc:nseg:l) = ei{j}(:,trc:nseg:l) / (K-1);
        ei{j}(:,wgt:nseg:l) = 1 ./ (exp(2*ei{j}(:,wgt:nseg:l)) + 1);

        if l == 0, % failed at the first run
            eei = [100*(K-1)/K*[1 1] 1];
        else
            eei = ei{j}(l-nseg+[tre tee trc]);
        end
        ee = [ee; [eei ite ei{j}(end)]];

        if l > max_l
            ec(1:end, end+1:l) = nan;
            max_l = l;
        end
        eci = repmat(nan, 1, max_l);
        eci(1:l) = ei{j}(1:l);
        ec = [ec; eci];
    end

    last = mean(ee(nmethod:nmethod:end, end-1));
end
if gs > 1 % pick the best from cross-validation
    assert(mod(nmethod, gs) == 0);
    nmethod = nmethod / gs;
    vale = zeros(gs, 1);
    best = [];
    for r = 1:folds:length(runs)
        ri = r:min(r+folds-1, length(runs));
        for i = 1:nmethod
            ii = ((ri-1) * nmethod + i-1) * gs;
            for g = 1:gs
                idx = ii + g;
                vale(g) = mean(ee(idx, tee), 1);
            end
            [mval, midx] = min(vale);
            best = [best ii+midx];
        end
    end
    best = sort(best);
    ee = ee(best, :); ec = ec(best, :);
end
assert(size(ee,1) == nmethod * length(runs));
assert(size(ec,1) == nmethod * length(runs));

for i = 1:nmethod
    ei = ee(i:nmethod:end, :);
    mee(i,:) = mean(ei, 1);
    ste(i,:) = std(ei) / sqrt(length(runs));
end

for i = 1:nmethod
    ei = ec(i:nmethod:end, :);
    c = isnan(ei); ei(c) = 0;
    se = sum(ei,1); sc = sum(~c,1);
    se(sc == 0) = nan; sc(sc == 0) = 1;
    mec(i,:) = se ./ sc;
end

if nargin > 3 % to save the data
    savedata([dat '.err'], mee, '%.8g');
    savedata([dat '.std'], ste, '%.8g');
    savedata([dat '.erc'], mec, '%.7g');
end
