Add tokenizer for OCaml

CONTRIBUTING.md

@@ -17,14 +17,14 @@ For a stable release, the commit specified in `vscodeRef` in [package.json](./pa
 
 Please understand that we only bundle languages with the monaco editor that have a significant relevance (for example, those that have an article in Wikipedia).
 
-- create `$/src/basic-languages/{myLang}/{myLang}.contribution.ts`
-- create `$/src/basic-languages/{myLang}/{myLang}.ts`
-- create `$/src/basic-languages/{myLang}/{myLang}.test.ts`
-- edit `$/src/basic-languages/monaco.contribution.ts` and register your new language
-- create `$/website/index/samples/sample.{myLang}.txt`
+- create `$/src/languages/definitions/{myLang}/register.ts`
+- create `$/src/languages/definitions/{myLang}/{myLang}.ts`
+- create `$/src/languages/definitions/{myLang}/{myLang}.test.ts`
+- edit `$/src/languages/definitions/register.all.ts` and register your new language
+- create `$/website/src/website/data/home-samples/sample.{myLang}.txt`
 
 ```js
-import './{myLang}/{myLang}.contribution';
+import './{myLang}/register';
 ```
 
 ## Debugging / Developing The Core Editor

src/languages/definitions/ocaml/ocaml.test.ts

@@ -0,0 +1,223 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('ocaml', [
+	// Comment
+	[
+		{
+			line: '(** documentation *)',
+			tokens: [
+				{ startIndex: 0, type: 'comment.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: ' (* comment (* nested comment *) *) ',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'comment.ocaml' },
+				{ startIndex: 35, type: '' }
+			]
+		}
+	],
+	[
+		{
+			line: ' (* comment (* nested comment *)',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'comment.ocaml' }
+			]
+		},
+		{
+			line: 'multiline comment',
+			tokens: [
+				{ startIndex: 0, type: 'comment.ocaml' }
+			]
+		},
+		{
+			line: ' *)',
+			tokens: [
+				{ startIndex: 0, type: 'comment.ocaml' }
+			]
+		},
+	],
+	// Integer
+	[
+		{
+			line: '37',
+			tokens: [
+				{ startIndex: 0, type: 'number.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '1_000_000',
+			tokens: [
+				{ startIndex: 0, type: 'number.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '0x00A9',
+			tokens: [
+				{ startIndex: 0, type: 'number.hex.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '0L',
+			tokens: [
+				{ startIndex: 0, type: 'number.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '3.141_592_653_589_793_12',
+			tokens: [
+				{ startIndex: 0, type: 'number.float.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '-1e-5',
+			tokens: [
+				{ startIndex: 0, type: 'number.float.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '0x1p-52',
+			tokens: [
+				{ startIndex: 0, type: 'number.float.ocaml' },
+			]
+		}
+	],
+	// Character
+	[
+		{
+			line: '\'a\'',
+			tokens: [
+				{ startIndex: 0, type: 'string.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '\'\\\'\'',
+			tokens: [
+				{ startIndex: 0, type: 'string.ocaml' },
+				{ startIndex: 1, type: 'string.escape.ocaml' },
+				{ startIndex: 3, type: 'string.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '\'\\xA9\'',
+			tokens: [
+				{ startIndex: 0, type: 'string.ocaml' },
+				{ startIndex: 1, type: 'string.escape.ocaml' },
+				{ startIndex: 5, type: 'string.ocaml' },
+			]
+		}
+	],
+	// String
+	[
+		{
+			line: '"Hello, World!\\n"',
+			tokens: [
+				{ startIndex: 0, type: 'string.ocaml' },
+				{ startIndex: 14, type: 'string.escape.ocaml' },
+				{ startIndex: 16, type: 'string.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '"\\u{207A}"',
+			tokens: [
+				{ startIndex: 0, type: 'string.ocaml' },
+				{ startIndex: 1, type: 'string.escape.ocaml' },
+				{ startIndex: 9, type: 'string.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: 'let longstr =',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.ocaml' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.ocaml' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'operator.ocaml' },
+			]
+		},
+		{
+			line: '  "Call me Ishmael. Some years ago — never mind how long \\',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'string.ocaml' },
+			]
+		},
+		{
+			line: '  precisely..."',
+			tokens: [
+				{ startIndex: 0, type: 'string.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: '{|"Hello, World!"|}',
+			tokens: [
+				{ startIndex: 0, type: 'string.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: 'let a = "This is a string"',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.ocaml' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.ocaml' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'operator.ocaml' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'string.ocaml' },
+			]
+		}
+	],
+	[
+		{
+			line: 'let pi = 4.0 *. atan 1.0',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.ocaml' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.ocaml' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'operator.ocaml' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'number.float.ocaml' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'operator.ocaml' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'identifier.ocaml' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'number.float.ocaml' }
+			]
+		}
+	],
+]);