Changed the way the FilteredCardList on the stats panel selects cards:

  used to: by running scripts on cards and comparing to string value of selected group
  now:     by keeping a list of group_ids for all cards, and comparing indices
Added 'bin size' attribute for making a histogram of numeric axes.
Added 'Text length' statistic.

data/magic.mse-game/game

@@ -522,6 +522,8 @@ init script:
 		else                                                   input
 	}
 	
+	word_count := break_text@(match:"[^[:space:]]+") + length
+
 # TODO : somewhere else?
 #card to conversion:
 #	name: MTGnews.com forums
@@ -1301,6 +1303,14 @@ statistics dimension:
 	script: stylesheet.short_name
 	icon: stats/stylesheet.png
 
+statistics dimension:
+	name: text length (words)
+	position hint: 100
+	script: word_count(to_text(card.rule_text))
+	numeric: true
+	bin size: 5
+	icon: stats/text_length.png
+
 statistics dimension:
 	name: race
 	position hint: 32

doc/type/statistics_dimension.txt

@@ -18,6 +18,8 @@ Categories are also automatically generated from dimensions.
 | @icon@		[[type:filename]]	 		Filename of an icon for this dimension.
 | @script@		[[type:script]]		''required''	Script that generates a value for each card in the set.
 | @numeric@		[[type:boolean]]	@false@		Is the value always a number?
+| @bin size@		[[type:double]]		''none''	For numeric dimensions: group numbers together into bins this large.<br/>
+			 			 		For example with @bin size: 5@, values @1@ and @3@ both get put under @"1-5"@.
 | @show empty@		[[type:boolean]]	@false@		Should cards with the value @""@ be included?
 | @split list@		[[type:boolean]]	@false@		Indicates the value is a list of the form @"item1, item2"@. The card is put under both items.
 | @colors@		[[type:map]] of opaque [[type:color]]s	 	Colors to use for specific values

src/data/statistics.cpp

@@ -23,6 +23,7 @@ StatsDimension::StatsDimension()
 	: automatic    (false)
 	, position_hint(0)
 	, numeric      (false)
+	, bin_size     (0)
 	, show_empty   (false)
 	, split_list   (false)
 {}
@@ -72,6 +73,7 @@ IMPLEMENT_REFLECTION_NO_GET_MEMBER(StatsDimension) {
 		REFLECT_N("icon", icon_filename);
 		REFLECT(script);
 		REFLECT(numeric);
+		REFLECT(bin_size);
 		REFLECT(show_empty);
 		REFLECT(split_list);
 		REFLECT(colors);

src/data/statistics.hpp

@@ -35,6 +35,7 @@ class StatsDimension : public IntrusivePtrBase<StatsDimension> {
 	Bitmap            icon;				///< The loaded icon (optional of course)
 	OptionalScript    script;			///< Script that determines the value(s)
 	bool              numeric;			///< Are the values numeric? If so, they require special sorting
+	double            bin_size;			///< Bin adjecent numbers?
 	bool              show_empty;		///< Should "" be shown?
 	bool              split_list;		///< Split values into multiple ones separated by commas
 	map<String,Color> colors;			///< Colors for the categories

src/gui/control/filtered_card_list.cpp

@@ -30,10 +30,14 @@ void FilteredCardList::onChangeSet() {
 
 void FilteredCardList::getItems(vector<VoidP>& out) const {
 	if (filter) {
-		FOR_EACH(c, set->cards) {
-			if (filter->keep(c)) {
-				out.push_back(c);
+		filter->getItems(set->cards,out);
 	}
+}
+
+void CardListFilter::getItems(const vector<CardP>& cards, vector<VoidP>& out) const {
+	FOR_EACH_CONST(c, cards) {
+		if (keep(c)) {
+			out.push_back(c);
 		}
 	}
 }

src/gui/control/filtered_card_list.hpp

@@ -21,7 +21,9 @@ class CardListFilter : public IntrusivePtrVirtualBase {
   public:
 	virtual ~CardListFilter() {}
 	/// Should a card be shown in the list?
-	virtual bool keep(const CardP& card) = 0;
+	virtual bool keep(const CardP& card) const { return false; }
+	/// Select cards from a card list
+	virtual void getItems(const vector<CardP>& cards, vector<VoidP>& out) const;
 };
 
 // ----------------------------------------------------------------------------- : FilteredCardList

src/gui/control/graph.cpp

@@ -16,6 +16,7 @@
 DECLARE_TYPEOF_COLLECTION(GraphAxisP);
 DECLARE_TYPEOF_COLLECTION(GraphElementP);
 DECLARE_TYPEOF_COLLECTION(GraphGroup);
+DECLARE_TYPEOF_COLLECTION(GraphDataElement*);
 DECLARE_TYPEOF_COLLECTION(GraphP);
 DECLARE_TYPEOF_COLLECTION(int);
 DECLARE_TYPEOF_COLLECTION(vector<int>);
@@ -32,20 +33,22 @@ DEFINE_EVENT_TYPE(EVENT_GRAPH_SELECT);
 // ----------------------------------------------------------------------------- : GraphAxis
 
 void GraphAxis::addGroup(const String& name, UInt size) {
+	if (!groups.empty() && groups.back().name == name) {
+		groups.back().size += size;
+	} else {
 		groups.push_back(GraphGroup(name, size));
-	max = std::max(max, size);
+	}
+	max = std::max(max, groups.back().size);
 	total += size;
 }
 
 // ----------------------------------------------------------------------------- : GraphData
 
-GraphElement::GraphElement(const String& v1) {
-	values.push_back(v1);
-}
-GraphElement::GraphElement(const String& v1, const String& v2) {
-	values.push_back(v1);
-	values.push_back(v2);
-}
+struct ComparingOriginalIndex {
+	inline bool operator () (const GraphElementP& a, const GraphElementP& b) {
+		return a->original_index < b->original_index;
+	}
+};
 
 void GraphDataPre::splitList(size_t axis) {
 	size_t count = elements.size(); // only the elements that were already there
@@ -63,6 +66,8 @@ void GraphDataPre::splitList(size_t axis) {
 			comma = v.find_first_of(_(','));
 		}
 	}
+	// re-sort by original_index
+	sort(elements.begin(), elements.end(), ComparingOriginalIndex());
 }
 
 
@@ -71,11 +76,25 @@ struct SmartLess{
 };
 DECLARE_TYPEOF(map<String COMMA UInt COMMA SmartLess>);
 
+String to_bin(double value, double bin_size) {
+	if (bin_size <= 0 || value == 0) {
+		return String() << (int)value;
+	} else {
+		int bin = ceil(value / bin_size);
+		return String::Format(_("%.0f%c%.0f"), (bin-1) * bin_size + 1, EN_DASH, bin * bin_size);
+	}
+}
+int bin_to_group(double value, double bin_size) {
+	if (bin_size <= 0 || value == 0) {
+		return 0;
+	} else {
+		return ceil(value / bin_size);
+	}
+}
+
 GraphData::GraphData(const GraphDataPre& d)
 	: axes(d.axes)
 {
-	// total size
-	size = (UInt)d.elements.size();
 	// find groups on each axis
 	size_t i = 0;
 	FOR_EACH(a, axes) {
@@ -92,19 +111,27 @@ GraphData::GraphData(const GraphDataPre& d)
 				double d;
 				if (c.first.ToDouble(&d)) {
 					// update mean
-					a->mean       += d * c.second;
+					a->mean_value += d * c.second;
+					a->max_value  = max(a->max_value, d);
 					numeric_count += c.second;
 					// add 0 bars before this value
 					int next = (int)floor(d);
 					for (int i = prev ; i < next ; i++) {
-						a->addGroup(String()<<i, 0);
+						a->addGroup(to_bin(i, a->bin_size), 0);
 					}
 					prev = next + 1;
-				}
 					// add
+					if (a->bin_size) {
+						a->addGroup(to_bin(d, a->bin_size), c.second);
+					} else {
+						a->addGroup(c.first, c.second);
+					}
+				} else {
+					// non-numeric, add anyway
 					a->addGroup(c.first, c.second);
 				}
-			a->mean /= numeric_count;
+			}
+			a->mean_value /= numeric_count;
 		} else if (a->order) {
 			// specific group order
 			FOR_EACH_CONST(gn, *a->order) {
@@ -145,34 +172,49 @@ GraphData::GraphData(const GraphDataPre& d)
 		++i;
 	}
 	// count elements in each position
-	values.clear();
+	values.reserve(d.elements.size());
+	size_t de_size = sizeof(GraphDataElement) + sizeof(int) * (axes.size() - 1);
 	FOR_EACH_CONST(e, d.elements) {
+		// make the group_nrs large enough
+		GraphDataElement* de = reinterpret_cast<GraphDataElement*>(new char[de_size]);
+		de->original_index = e->original_index;
 		// find index j in elements
-		vector<int> group_nrs(axes.size(), -1);
 		int i = 0;
 		FOR_EACH(a, axes) {
 			String v = e->values[i];
+			de->group_nrs[i] = -1;
+			double d;
+			if (a->numeric && a->bin_size > 0 && v.ToDouble(&d)) {
+				// calculate group that contains v
+				de->group_nrs[i] = bin_to_group(d, a->bin_size);
+			} else {
+				// find group that contains v
 				int j = 0;
 				FOR_EACH(g, a->groups) {
 					if (v == g.name) {
-					group_nrs[i] = j;
+						de->group_nrs[i] = j;
 						break;
 					}
 					++j;
 				}
+			}
 			++i;
 		}
-		values.push_back(group_nrs);
+		values.push_back(de);
 	}
 }
 
+GraphData::~GraphData() {
+	FOR_EACH_CONST(v,values) delete v;
+}
+
 void GraphData::crossAxis(size_t axis1, size_t axis2, vector<UInt>& out) const {
 	size_t a1_size = axes[axis1]->groups.size();
 	size_t a2_size = axes[axis2]->groups.size();
 	out.clear();
 	out.resize(a1_size * a2_size, 0);
 	FOR_EACH_CONST(v, values) {
-		int v1 = v[axis1], v2 = v[axis2];
+		int v1 = v->group_nrs[axis1], v2 = v->group_nrs[axis2];
 		if (v1 >= 0 && v2 >= 0) {
 			out[a2_size * v1 + v2]++;
 		}
@@ -186,29 +228,46 @@ void GraphData::crossAxis(size_t axis1, size_t axis2, size_t axis3, vector<UInt>
 	out.clear();
 	out.resize(a1_size * a2_size * a3_size, 0);
 	FOR_EACH_CONST(v, values) {
-		int v1 = v[axis1], v2 = v[axis2], v3 = v[axis3];
+		int v1 = v->group_nrs[axis1], v2 = v->group_nrs[axis2], v3 = v->group_nrs[axis3];
 		if (v1 >= 0 && v2 >= 0 && v3 >= 0) {
 			out[a3_size * (a2_size * v1 + v2) + v3]++;
 		}
 	}
 }
 
+bool matches(const GraphDataElement* v, const vector<int>& match) {
+	for (size_t i = 0 ; i < match.size() ; ++i) {
+		if (v->group_nrs[i] == -1 || match[i] != -1 && v->group_nrs[i] != match[i]) {
+			return false;
+		}
+	}
+	return true;
+}
+
 UInt GraphData::count(const vector<int>& match) const {
 	if (match.size() != axes.size()) return 0;
 	UInt count = 0;
+	size_t prev_index = (size_t)-1;
 	FOR_EACH_CONST(v, values) {
-		bool matches = true;
-		for (size_t i = 0 ; i < match.size() ; ++i) {
-			if (v[i] == -1 || match[i] != -1 && v[i] != match[i]) {
-				matches = false;
-				break;
-			}
+		if (matches(v, match) && v->original_index != prev_index) {
+			prev_index = v->original_index; // don't count the same index twice
+			count += matches(v, match);
 		}
-		count += matches;
 	}
 	return count;
 }
 
+void GraphData::indices(const vector<int>& match, vector<size_t>& out) const {
+	if (match.size() != axes.size()) return;
+	size_t prev_index = (size_t)-1;
+	FOR_EACH_CONST(v, values) {
+		if (matches(v, match) && v->original_index != prev_index) {
+			prev_index = v->original_index; // don't select the same index twice
+			out.push_back(v->original_index);
+		}
+	}
+}
+
 // ----------------------------------------------------------------------------- : Graph1D
 
 void Graph1D::draw(RotatedDC& dc, const vector<int>& current, DrawLayer layer) const {
@@ -620,8 +679,8 @@ void GraphStats::setData(const GraphDataP& d) {
 	values.clear();
 	if (!axis.numeric) return;
 	if (axis.groups.empty()) return;
-	values.push_back(make_pair(_("max"),  axis.groups.back().name));
-	values.push_back(make_pair(_("mean"), String::Format(_("%.2f"), axis.mean)));
+	values.push_back(make_pair(_("max"),  String::Format(_("%.2f"), axis.max_value)));
+	values.push_back(make_pair(_("mean"), String::Format(_("%.2f"), axis.mean_value)));
 }
 
 RealSize GraphStats::determineSize(RotatedDC& dc) const {
@@ -978,6 +1037,10 @@ void GraphControl::setData(const GraphDataP& data) {
 	}
 	Refresh(false);
 }
+GraphDataP GraphControl::getData() const {
+	if (graph) return graph->getData();
+	else       return GraphDataP();
+}
 
 size_t GraphControl::getDimensionality() const {
 	if (graph) return graph->getData()->axes.size();
@@ -1067,6 +1130,9 @@ String GraphControl::getSelection(size_t axis) const {
 	if (i == -1 || (size_t)i >= a.groups.size()) return wxEmptyString;
 	return a.groups[current_item[axis]].name;
 }
+vector<int> GraphControl::getSelectionIndices() const {
+	return current_item;
+}
 
 void GraphControl::onMotion(wxMouseEvent& ev) {
 	if (!graph) return;

src/gui/control/graph.hpp

@@ -52,13 +52,13 @@ enum AutoColor
 /** The sum of groups.sum = sum of all elements in the data */
 class GraphAxis : public IntrusivePtrBase<GraphAxis> {
   public:
-	GraphAxis(const String& name, AutoColor auto_color = AUTO_COLOR_EVEN, bool numeric = false, const map<String,Color>* colors = nullptr, const vector<String>* order = nullptr)
+	GraphAxis(const String& name, AutoColor auto_color = AUTO_COLOR_EVEN, bool numeric = false, double bin_size = 0, const map<String,Color>* colors = nullptr, const vector<String>* order = nullptr)
 		: name(name)
 		, auto_color(auto_color)
-		, numeric(numeric)
+		, numeric(numeric), bin_size(bin_size)
 		, max(0)
 		, total(0)
-		, mean(0)
+		, mean_value(0), max_value(-numeric_limits<double>::infinity())
 		, colors(colors)
 		, order(order)
 	{}
@@ -67,9 +67,11 @@ class GraphAxis : public IntrusivePtrBase<GraphAxis> {
 	AutoColor          auto_color;	///< Automatically assign colors to the groups on this axis
 	vector<GraphGroup> groups;		///< Groups along this axis
 	bool               numeric;		///< Numeric axis?
+	double             bin_size;	///< Group numeric values into bins of this size
 	UInt               max;			///< Maximum size of the groups
 	UInt               total;		///< Sum of the size of all groups
-	double             mean;		///< Mean value, only for numeric axes
+	double             mean_value;		///< Mean value, only for numeric axes
+	double             max_value;		///< Maximal value, only for numeric axes
 	const map<String,Color>* colors;	///< Colors for each choice (optional)
 	const vector<String>*    order;		///< Order of the items (optional)
 	
@@ -80,10 +82,9 @@ class GraphAxis : public IntrusivePtrBase<GraphAxis> {
 /// A single data point of a graph
 class GraphElement : public IntrusivePtrBase<GraphElement> {
   public:
-	GraphElement() {}
-	GraphElement(const String& v1);
-	GraphElement(const String& v1, const String& v2);
+	GraphElement(size_t original_index) : original_index(original_index) {}
 	
+	size_t         original_index; ///< Corresponding index in the original input
 	vector<String> values;         ///< Group name for each axis
 };
 
@@ -96,13 +97,20 @@ class GraphDataPre {
 	void splitList(size_t axis);
 };
 
+/// A single data point of a graph
+struct GraphDataElement {
+	size_t original_index;
+	int    group_nrs[1];   ///< Group number for each axis
+};
+
 /// Data to be displayed in a graph
 class GraphData : public IntrusivePtrBase<GraphData> {
   public:
 	GraphData(const GraphDataPre&);
+	~GraphData();
 	
 	vector<GraphAxisP>        axes;		///< The axes in the data
-	vector<vector<int> > values;	///< All elements, with the group number for each axis, or -1
+	vector<GraphDataElement*> values;	///< All elements, with the group number for each axis, or -1
 	UInt                      size;		///< Total number of elements
 	
 	/// Create a cross table for two axes
@@ -111,6 +119,8 @@ class GraphData : public IntrusivePtrBase<GraphData> {
 	void crossAxis(size_t axis1, size_t axis2, size_t axis3, vector<UInt>& out) const;
 	/// Count the number of elements with the given values, -1 is a wildcard
 	UInt count(const vector<int>& match) const;
+	/// Get the original_indices of elements matching the selection
+	void indices(const vector<int>& match, vector<size_t>& out) const;
 };
 
 
@@ -333,11 +343,15 @@ class GraphControl : public wxControl {
 	void setData(const GraphDataPre& data);
 	/// Update the data in the graph
 	void setData(const GraphDataP& data);
+	/// Retrieve the data in the graph
+	GraphDataP getData() const;
 	
 	/// Is there a selection on the given axis?
 	bool hasSelection(size_t axis) const;
 	/// Get the current item along the given axis
 	String getSelection(size_t axis) const;
+	/// Get the current item along each axis
+	vector<int> getSelectionIndices() const;
 	
 	/// Get the current layout
 	GraphType getLayout() const;

src/gui/set/stats_panel.cpp

@@ -23,6 +23,7 @@
 
 DECLARE_TYPEOF_COLLECTION(StatsDimensionP);
 DECLARE_TYPEOF_COLLECTION(String);
+DECLARE_TYPEOF_COLLECTION(size_t);
 DECLARE_TYPEOF_COLLECTION(CardP);
 typedef pair<StatsDimensionP,String> pair_StatsDimensionP_String;
 DECLARE_TYPEOF_COLLECTION(pair_StatsDimensionP_String);
@@ -80,6 +81,7 @@ void StatCategoryList::drawItem(DC& dc, int x, int y, size_t item) {
 	if (!cat.icon_filename.empty() && !cat.icon.Ok()) {
 		InputStreamP file = game->openIn(cat.icon_filename);
 		Image img(*file);
+		if (img.HasMask()) img.InitAlpha(); // we can't handle masks
 		if (img.Ok()) {
 			cat.icon = Bitmap(resample_preserve_aspect(img, 21, 21));
 		}
@@ -236,6 +238,7 @@ void StatDimensionList::drawItem(DC& dc, int x, int y, size_t item) {
 	if (!dim.icon_filename.empty() && !dim.icon.Ok()) {
 		InputStreamP file = game->openIn(dim.icon_filename);
 		Image img(*file);
+		if (img.HasMask()) img.InitAlpha(); // we can't handle masks
 		Image resampled(21, 21);
 		resample_preserve_aspect(img, resampled);
 		if (img.Ok()) dim.icon = Bitmap(resampled);
@@ -394,32 +397,7 @@ void StatsPanel::onCommand(int id) {
 	}
 }
 
-// ----------------------------------------------------------------------------- : Filtering card list
-
-bool chosen(const String& choice, const String& input);
-
-class StatsFilter : public CardListFilter {
-  public:
-	StatsFilter(Set& set)
-		: set(set)
-	{}
-	virtual bool keep(const CardP& card) {
-		Context& ctx = set.getContext(card);
-		FOR_EACH(v, values) {
-			StatsDimension& dim = *v.first;
-			String value = untag(dim.script.invoke(ctx)->toString());
-			if (dim.split_list) {
-				if (!chosen(v.second, value)) return false;
-			} else {
-				if (value != v.second) return false;
-			}
-		}
-		return true;
-	}
-	
-	vector<pair<StatsDimensionP, String> > values; ///< Values selected along each dimension
-	Set& set;
-};
+// ----------------------------------------------------------------------------- : Updating graph
 
 void StatsPanel::onChange() {
 	if (active) {
@@ -467,19 +445,20 @@ void StatsPanel::showCategory(const GraphType* prefer_layout) {
 	// create axes
 	GraphDataPre d;
 	FOR_EACH(dim, dims) {
-		d.axes.push_back(new_intrusive5<GraphAxis>(
+		d.axes.push_back(new_intrusive6<GraphAxis>(
 			dim->name,
 			dim->colors.empty() ? AUTO_COLOR_EVEN : AUTO_COLOR_NO,
 			dim->numeric,
+			dim->bin_size,
 			&dim->colors,
 			dim->groups.empty() ? nullptr : &dim->groups
 			)
 		);
 	}
-	// find values
-	FOR_EACH(card, set->cards) {
-		Context& ctx = set->getContext(card);
-		GraphElementP e(new GraphElement);
+	// find values for each card
+	for (size_t i = 0 ; i < set->cards.size() ; ++i) {
+		Context& ctx = set->getContext(set->cards[i]);
+		GraphElementP e(new GraphElement(i));
 		bool show = true;
 		FOR_EACH(dim, dims) {
 			String value = untag(dim->script.invoke(ctx)->toString());
@@ -517,37 +496,34 @@ void StatsPanel::showLayout(GraphType layout) {
 	graph->setLayout(layout);
 	graph->Refresh(false);
 }
+
 void StatsPanel::onGraphSelect(wxCommandEvent&) {
 	filterCards();
 }
 
-void StatsPanel::filterCards() {
-	#if USE_SEPARATE_DIMENSION_LISTS
-		vector<StatsDimensionP> dims;
-		for (int i = 0 ; i < 3 ; ++i) {
-			StatsDimensionP dim = dimensions[i]->getSelection();
-			if (dim) dims.push_back(dim);
-		}
-	#elif USE_DIMENSION_LISTS
-		vector<StatsDimensionP> dims;
-		for (size_t i = 0 ; i < dimensions->prefered_dimension_count ; ++i) {
-			StatsDimensionP dim = dimensions->getSelection(i);
-			if (dim) dims.push_back(dim);
+// ----------------------------------------------------------------------------- : Filtering card list
+
+class StatsFilter : public CardListFilter {
+  public:
+	StatsFilter(GraphData& data, const vector<int> match) {
+		data.indices(match, indices);
 	}
-	#else
-		if (!categories->hasSelection()) return;
-		const StatsCategory& cat = categories->getSelection();
-		const vector<StatsDimensionP>& dims = cat.dimensions;
-	#endif
-	intrusive_ptr<StatsFilter> filter(new StatsFilter(*set));
-	for (size_t i = 0 ; i < dims.size() ; ++i) {
-		if (graph->hasSelection(i)) {
-			filter->values.push_back(make_pair(dims[i], graph->getSelection(i)));
+	virtual void getItems(const vector<CardP>& cards, vector<VoidP>& out) const {
+		FOR_EACH_CONST(idx, indices) {
+			out.push_back(cards.at(idx));
 		}
 	}
+	
+	vector<size_t> indices; ///< Indices of cards to select
+};
+
+void StatsPanel::filterCards() {
+	intrusive_ptr<StatsFilter> filter(new StatsFilter(*graph->getData(), graph->getSelectionIndices()));
 	card_list->setFilter(filter);
 }
 
+// ----------------------------------------------------------------------------- : Events
+
 BEGIN_EVENT_TABLE(StatsPanel, wxPanel)
 	EVT_GRAPH_SELECT(wxID_ANY, StatsPanel::onGraphSelect)
 END_EVENT_TABLE()

src/util/string.hpp

@@ -68,6 +68,8 @@ void writeUTF8(wxTextOutputStream& stream, const String& str);
 	#define RIGHT_SINGLE_QUOTE  _('\u2019')
 	#define  LEFT_DOUBLE_QUOTE  _('\u201C')
 	#define RIGHT_DOUBLE_QUOTE  _('\u201D')
+	#define EN_DASH             _('\u2013')
+	#define EM_DASH             _('\u2014')
 	#define CONNECTION_SPACE    _('\uEB00') // in private use area, untags to ' '
 #else
 	#define  LEFT_ANGLE_BRACKET _("<")
@@ -76,6 +78,8 @@ void writeUTF8(wxTextOutputStream& stream, const String& str);
 	#define RIGHT_SINGLE_QUOTE  _('\'')
 	#define  LEFT_DOUBLE_QUOTE  _('\"')
 	#define RIGHT_DOUBLE_QUOTE  _('\"')
+	#define EN_DASH             _('-') // 150?
+	#define EM_DASH             _('-') // 151?
 	#define CONNECTION_SPACE    _(' ') // too bad
 #endif