Improved i18n key input for X11. Which means languages like cyrillic work now.

Japanese not yet, but we're even a little closer to that now.
Dead-key handling is not working. I just can't figure it out (X11 documentation and examples make me swear badly).
Users which have language input with dead-keys (like ^) will now have those characters always ignored.
Before it had just printed the characters but set a complete wrong key-code. So dead keys partly better, partly worse.
But this solution is going in the right direction. Either we figure out later why X11 doesn't handle dead-keys or we handle them ourselves.


git-svn-id: svn://svn.code.sf.net/p/irrlicht/code/trunk@4743 dfc29bdd-3216-0410-991c-e03cc46cb475

changes.txt

 --------------------------
 Changes in 1.9 (not yet released)
 
+- Improved i18n key input for X11 (languages like cyrillic work now).
 - Fix bug that ListBox would not allow to 'tab' to the next element (thx @ FlavourBoat for reporting)
 - IGUIEnvironment::getNextElement now public (was only in CGUIEnvironment before).
 - Add an overwrite mode to editbox. Patch provided by Adam(aka kingadami).
@@ -12,7 +13,7 @@ Changes in 1.9 (not yet released)
 - Fix c::b project obj folder names. Some static builds had used the shared folders (thx @ gerdb for reporting)
 - Add ITexture::getSource which can be used to check where the last IVideoDriver::getTexture call found the texture.
 - Add IMeshTextureLoader interface and replace texture-loading algorithms in most meshloaders.
-- CGUICheckBox no longer gives up focus on EMIE_LMOUSE_LEFT_UP (thx @Demre for reporting)
+- CGUICheckBox no longer gives up focus on EMIE_LMOUSE_LEFT_UP (thx @Demre and @REDDemon for reporting)
 - Bugfix: IGUIElement::addChild now prevents setting an element as it's own child.
 - GUI editor improvements (prevent crash, improve UI)
 - Add IrrlichtDevice::setWindowSize. 

include/IEventReceiver.h

@@ -315,7 +315,7 @@ struct SEvent
 	//! Any kind of keyboard event.
 	struct SKeyInput
 	{
-		//! Character corresponding to the key (0, if not a character)
+		//! Character corresponding to the key (0, if not a character, value undefined in key releases)
 		wchar_t Char;
 
 		//! Key which has been pressed or released

source/Irrlicht/CIrrDeviceLinux.cpp

@@ -10,6 +10,7 @@
 #include <stdlib.h>
 #include <sys/utsname.h>
 #include <time.h>
+#include <locale.h>  
 #include "IEventReceiver.h"
 #include "ISceneManager.h"
 #include "IGUIEnvironment.h"
@@ -75,6 +76,7 @@ CIrrDeviceLinux::CIrrDeviceLinux(const SIrrlichtCreationParameters& param)
 	: CIrrDeviceStub(param),
 #ifdef _IRR_COMPILE_WITH_X11_
 	display(0), visual(0), screennr(0), window(0), StdHints(0), SoftwareImage(0),
+	XInputMethod(0), XInputContext(0),
 #ifdef _IRR_COMPILE_WITH_OPENGL_
 	glxWin(0),
 	Context(0),
@@ -126,6 +128,10 @@ CIrrDeviceLinux::CIrrDeviceLinux(const SIrrlichtCreationParameters& param)
 	if (!VideoDriver)
 		return;
 
+#ifdef _IRR_COMPILE_WITH_X11_
+	createInputContext();
+#endif
+	
 	createGUIAndScene();
 }
 
@@ -160,6 +166,8 @@ CIrrDeviceLinux::~CIrrDeviceLinux()
 		VideoDriver = NULL;
 	}
 
+	destroyInputContext();
+	
 	if (display)
 	{
 		#ifdef _IRR_COMPILE_WITH_OPENGL_
@@ -867,6 +875,123 @@ void CIrrDeviceLinux::createDriver()
 	}
 }
 
+#ifdef _IRR_COMPILE_WITH_X11_	
+bool CIrrDeviceLinux::createInputContext()
+{
+	// One one side it would be nicer to let users do that - on the other hand
+	// not setting the environment locale will not work when using i18n X11 functions.
+	// So users would have to call it always or their input is broken badly.
+	// We can restore immediately - so shouldn't mess with anything in users apps.
+	core::stringc oldLocale(setlocale(LC_CTYPE, NULL));	
+	setlocale(LC_CTYPE, "");	// use environmenbt locale
+	
+	if ( !XSupportsLocale() )
+	{
+		os::Printer::log("Locale not supported. Falling back to non-i18n input.", ELL_WARNING);
+		setlocale(LC_CTYPE, oldLocale.c_str());
+		return false;
+	}
+	
+	XInputMethod = XOpenIM(display, NULL, NULL, NULL);
+	if ( !XInputMethod )
+	{
+		setlocale(LC_CTYPE, oldLocale.c_str());
+		os::Printer::log("XOpenIM failed to create an input method. Falling back to non-i18n input.", ELL_WARNING);
+		return false;
+	}
+
+	XIMStyles *im_supported_styles;
+	XGetIMValues(XInputMethod, XNQueryInputStyle, &im_supported_styles, (char*)NULL);
+	XIMStyle bestStyle = 0;
+	// TODO: If we want to support languages like chinese or japanese as well we probably have to work with callbacks here.
+	XIMStyle supportedStyle = XIMPreeditNone | XIMStatusNone;	
+    for(int i=0; i < im_supported_styles->count_styles; ++i) 
+	{
+        XIMStyle style = im_supported_styles->supported_styles[i];
+        if ((style & supportedStyle) == style) /* if we can handle it */
+		{
+            bestStyle = style;
+			break;
+		}
+    }
+	XFree(im_supported_styles);
+	
+	if ( !bestStyle )
+	{
+		XDestroyIC(XInputContext);
+		XInputContext = 0;
+		
+		os::Printer::log("XInputMethod has no input style we can use. Falling back to non-i18n input.", ELL_WARNING);
+		setlocale(LC_CTYPE, oldLocale.c_str());
+		return false;
+	}
+
+	XInputContext = XCreateIC(XInputMethod, 
+							XNInputStyle, bestStyle, 
+							XNClientWindow, window,
+							(char*)NULL);
+	if (!XInputContext )
+	{
+		os::Printer::log("XInputContext failed to create an input context. Falling back to non-i18n input.", ELL_WARNING);
+		setlocale(LC_CTYPE, oldLocale.c_str());
+		return false;
+	}
+	XSetICFocus(XInputContext);
+	setlocale(LC_CTYPE, oldLocale.c_str());
+	return true;
+}
+
+void CIrrDeviceLinux::destroyInputContext()
+{
+	if ( XInputContext )
+	{
+		XUnsetICFocus(XInputContext);
+		XDestroyIC(XInputContext);
+		XInputContext = 0;
+	}
+	if ( XInputMethod )
+	{
+		XCloseIM(XInputMethod);
+		XInputMethod = 0;
+	}
+}
+
+EKEY_CODE CIrrDeviceLinux::getKeyCode(XEvent &event)
+{
+	EKEY_CODE keyCode = (EKEY_CODE)0;
+	
+	SKeyMap mp;
+	mp.X11Key = XkbKeycodeToKeysym(display, event.xkey.keycode, 0, 0);
+	// mp.X11Key = XKeycodeToKeysym(display, event.xkey.keycode, 0);	// deprecated, if we still find platforms which need that we have to use some define
+	const s32 idx = KeyMap.binary_search(mp);
+	if (idx != -1)
+	{
+		keyCode = (EKEY_CODE)KeyMap[idx].Win32Key;
+	}
+	if (keyCode == 0)
+	{
+		// Any value is better than none, that allows at least using the keys.
+		// Worst case is that some keys will be identical, still better than _all_
+		// unknown keys being identical.		
+		if ( !mp.X11Key )
+		{
+			keyCode = (EKEY_CODE)event.xkey.keycode; 
+			os::Printer::log("No such X11Key, using event keycode", core::stringc(event.xkey.keycode).c_str(), ELL_INFORMATION);
+		}
+		else if (idx == -1)
+		{
+			keyCode = (EKEY_CODE)mp.X11Key;
+			os::Printer::log("EKEY_CODE not found, using orig. X11 keycode", core::stringc(mp.X11Key).c_str(), ELL_INFORMATION);
+		}
+		else
+		{
+			keyCode = (EKEY_CODE)mp.X11Key;
+			os::Printer::log("EKEY_CODE is 0, using orig. X11 keycode", core::stringc(mp.X11Key).c_str(), ELL_INFORMATION);
+		}
+ 	}
+	return keyCode;
+}
+#endif	
 
 //! runs the device. Returns false if device wants to be deleted
 bool CIrrDeviceLinux::run()
@@ -1043,52 +1168,68 @@ bool CIrrDeviceLinux::run()
 						(next_event.xkey.keycode == event.xkey.keycode) &&
 						(next_event.xkey.time - event.xkey.time) < 2)	// usually same time, but on some systems a difference of 1 is possible
 					{
-						/* Ignore the key release event */
+						// Ignore the key release event
 						break;
 					}
 				}
-				// fall-through in case the release should be handled
-			case KeyPress:
-				{
-					SKeyMap mp;
-					char buf[8]={0};
-					XLookupString(&event.xkey, buf, sizeof(buf), &mp.X11Key, NULL);
 
-					irrevent.EventType = irr::EET_KEY_INPUT_EVENT;
-					irrevent.KeyInput.PressedDown = (event.type == KeyPress);
-//					mbtowc(&irrevent.KeyInput.Char, buf, sizeof(buf));
-					irrevent.KeyInput.Char = ((wchar_t*)(buf))[0];
-					irrevent.KeyInput.Control = (event.xkey.state & ControlMask) != 0;
-					irrevent.KeyInput.Shift = (event.xkey.state & ShiftMask) != 0;
+				irrevent.EventType = irr::EET_KEY_INPUT_EVENT;
+				irrevent.KeyInput.PressedDown = false;
+				irrevent.KeyInput.Char = 0;	// on release that's undefined
+				irrevent.KeyInput.Control = (event.xkey.state & ControlMask) != 0;
+				irrevent.KeyInput.Shift = (event.xkey.state & ShiftMask) != 0;
+				irrevent.KeyInput.Key = getKeyCode(event);
 
-					event.xkey.state &= ~(ControlMask|ShiftMask); // ignore shift-ctrl states for figuring out the key
-					XLookupString(&event.xkey, buf, sizeof(buf), &mp.X11Key, NULL);
-					const s32 idx = KeyMap.binary_search(mp);
-					if (idx != -1)
-					{
-						irrevent.KeyInput.Key = (EKEY_CODE)KeyMap[idx].Win32Key;
-					}
-					else
-					{
-						irrevent.KeyInput.Key = (EKEY_CODE)0;
-					}
-					if (irrevent.KeyInput.Key == 0)
+				postEventFromUser(irrevent);
+				break;
+
+			case KeyPress:
+				{
+					SKeyMap mp;					
+					if ( XInputContext )
 					{
-						// 1:1 mapping to windows-keys would require testing for keyboard type (us, ger, ...)
-						// So unless we do that we will have some unknown keys here.
-						if (idx == -1)
+						wchar_t buf[8]={0};
+						Status status;
+						int strLen = XwcLookupString(XInputContext, &event.xkey, buf, sizeof(buf), &mp.X11Key, &status);
+						if ( status == XBufferOverflow )
 						{
-							os::Printer::log("Could not find EKEY_CODE, using orig. X11 keycode instead", core::stringc(event.xkey.keycode).c_str(), ELL_INFORMATION);
+							os::Printer::log("XwcLookupString needs a larger buffer", ELL_INFORMATION);
+						}
+						if ( strLen > 0 && (status == XLookupChars || status == XLookupBoth) )
+						{
+							if ( strLen > 1 )
+								os::Printer::log("Additional returned characters dropped", ELL_INFORMATION);
+							irrevent.KeyInput.Char = buf[0];
 						}
 						else
 						{
-							os::Printer::log("EKEY_CODE is 0, using orig. X11 keycode instead", core::stringc(event.xkey.keycode).c_str(), ELL_INFORMATION);
+#if 0 // Most of those are fine - but useful to have the info when debugging Irrlicht itself.
+							if ( status == XLookupNone )
+								os::Printer::log("XLookupNone", ELL_INFORMATION);
+							else if ( status ==  XLookupKeySym )	
+								// Getting this also when user did not set setlocale(LC_ALL, ""); and using an unknown locale
+								// XSupportsLocale doesn't seeem to catch that unfortunately - any other ideas to catch it are welcome.
+								os::Printer::log("XLookupKeySym", ELL_INFORMATION);
+							else if ( status ==  XBufferOverflow )
+								os::Printer::log("XBufferOverflow", ELL_INFORMATION);
+							else if ( strLen == 0 )
+								os::Printer::log("no string", ELL_INFORMATION);
+#endif
+							irrevent.KeyInput.Char = 0;
 						}
-						// Any value is better than none, that allows at least using the keys.
-						// Worst case is that some keys will be identical, still better than _all_
-						// unknown keys being identical.
-						irrevent.KeyInput.Key = (EKEY_CODE)event.xkey.keycode;
 					}
+					else	// Old version without InputContext. Does not support i18n, but good to have as fallback.
+					{
+						char buf[8]={0};
+						XLookupString(&event.xkey, buf, sizeof(buf), &mp.X11Key, NULL);
+						irrevent.KeyInput.Char = ((wchar_t*)(buf))[0];
+					}
+					
+					irrevent.EventType = irr::EET_KEY_INPUT_EVENT;
+					irrevent.KeyInput.PressedDown = true;
+					irrevent.KeyInput.Control = (event.xkey.state & ControlMask) != 0;
+					irrevent.KeyInput.Shift = (event.xkey.state & ShiftMask) != 0;
+					irrevent.KeyInput.Key = getKeyCode(event);
 
 					postEventFromUser(irrevent);
 				}
@@ -1474,6 +1615,7 @@ void CIrrDeviceLinux::createKeyMap()
 	// I don't know if this is the best method  to create
 	// the lookuptable, but I'll leave it like that until
 	// I find a better version.
+	// Search for missing numbers in keysymdef.h
 
 #ifdef _IRR_COMPILE_WITH_X11_
 	KeyMap.reallocate(84);
@@ -1629,10 +1771,13 @@ void CIrrDeviceLinux::createKeyMap()
 	KeyMap.push_back(SKeyMap(XK_backslash, KEY_OEM_5));
 	KeyMap.push_back(SKeyMap(XK_bracketright, KEY_OEM_6));
 	KeyMap.push_back(SKeyMap(XK_asciicircum, KEY_OEM_5));
+	KeyMap.push_back(SKeyMap(XK_dead_circumflex, KEY_OEM_5));	
 	KeyMap.push_back(SKeyMap(XK_degree, 0)); //?
 	KeyMap.push_back(SKeyMap(XK_underscore, KEY_MINUS)); //?
 	KeyMap.push_back(SKeyMap(XK_grave, KEY_OEM_3));
+	KeyMap.push_back(SKeyMap(XK_dead_grave, KEY_OEM_3));
 	KeyMap.push_back(SKeyMap(XK_acute, KEY_OEM_6));
+	KeyMap.push_back(SKeyMap(XK_dead_acute, KEY_OEM_6));
 	KeyMap.push_back(SKeyMap(XK_a, KEY_KEY_A));
 	KeyMap.push_back(SKeyMap(XK_b, KEY_KEY_B));
 	KeyMap.push_back(SKeyMap(XK_c, KEY_KEY_C));

source/Irrlicht/CIrrDeviceLinux.h

@@ -156,6 +156,12 @@ namespace irr
 		void initXAtoms();
 
 		bool switchToFullscreen(bool reset=false);
+	
+#ifdef _IRR_COMPILE_WITH_X11_	
+		bool createInputContext();
+		void destroyInputContext();
+		EKEY_CODE getKeyCode(XEvent &event);
+#endif	
 
 		//! Implementation of the linux cursor control
 		class CCursorControl : public gui::ICursorControl
@@ -394,6 +400,8 @@ namespace irr
 		XSetWindowAttributes attributes;
 		XSizeHints* StdHints;
 		XImage* SoftwareImage;
+		XIM XInputMethod;
+		XIC XInputContext;
 		mutable core::stringc Clipboard;
 		#ifdef _IRR_LINUX_X11_VIDMODE_
 		XF86VidModeModeInfo oldVideoMode;