Patrick Rein uploaded a new version of MultilingualTests to project The Trunk:
http://source.squeak.org/trunk/MultilingualTests-pre.24.mcz

==================== Summary ====================

Name: MultilingualTests-pre.24
Author: pre
Time: 8 June 2017, 9:44:45.939697 am
UUID: 04c3d62b-8671-ff46-abc0-9704cb38be0c
Ancestors: MultilingualTests-ul.23

Adds more extensive tests for encodings. Fixes MultiByteFileStreamTest broken due to abstract encoding classes.

=============== Diff against MultilingualTests-ul.23 ===============

Item was changed:
  ----- Method: MultiByteFileStreamTest>>testByteTextConverter (in category 'testing') -----
  testByteTextConverter
+ 	| strings converterClasses |
+ 	strings := {
+ 		String newFrom: ((0 to: 255) collect: [:e | e asCharacter]).
+ 	}.
+ 	
+ 	converterClasses := ByteTextConverter allSubclasses 
+ 		copyWithoutAll: {ISO8859TextConverter . WinCPTextConverter . MacOSCPTextConverter}.
+ 	converterClasses do: [:converterClass | 
+ 		strings do: [:string | | converter stream encoded decoded encoded2 |
+ 			converter := converterClass new.
+ 			stream := string readStream.
+ 			encoded := string select: [:e | (converter nextFromStream: stream) notNil].
+ 			stream := encoded readStream.
+ 			decoded := encoded collect: [:e | converter nextFromStream: stream].
+ 			self assert: stream atEnd.
+ 			stream := String new writeStream.
+ 			converter nextPutAll: decoded toStream: stream.
+ 			encoded2 := stream contents.
+ 			self assert: (encoded2 collect: [:e | e charCode] as: Array) = (encoded collect: [:e | e charCode] as: Array).
+ 			stream := String new writeStream.
+ 			decoded do: [:e | converter nextPut: e toStream: stream].
+ 			encoded2 := stream contents.
+ 			self assert: (encoded2 collect: [:e | e charCode] as: Array) = (encoded collect: [:e | e charCode] as: Array)]]!
- 
- 	| byteCharacters |
- 	byteCharacters := Character allByteCharacters.
- 	ByteTextConverter allSubclassesDo: [ :converterClass |
- 		| converter stream encoded decoded encoded2 |
- 		converter := converterClass new.
- 		stream := byteCharacters readStream.
- 		"Find bytes that can be decoded by this converter."
- 		encoded := byteCharacters select: [ :e | (converter nextFromStream: stream) notNil ].
- 		"Decode those bytes."
- 		stream := encoded readStream.
- 		decoded := encoded collect: [ :e | converter nextFromStream: stream ].
- 		self assert: stream atEnd.
- 		"Re-encode the decoded bytes using #nextPutAll:toStream:, and check if they match the original bytes."
- 		encoded2 := String streamContents: [ :writeStream |
- 			converter nextPutAll: decoded toStream: writeStream ].
- 		encoded with: encoded2 do: [ :original :reencoded |
- 			self 
- 				assert: original charCode = reencoded charCode
- 				description: [ '{1} could not decode or re-encode {2} using #nextPutAll:toStream:. Instead, it yielded {3}.'
- 					format: {
- 						converterClass name.
- 						original charCode.
- 						reencoded charCode } ] ].
- 		"Re-encode the decoded bytes using #nextPut:toStream:, and check if they match the original bytes."
- 		encoded2 := String streamContents: [ :writeStream |
- 			decoded do: [:e | converter nextPut: e toStream: writeStream ] ].
- 		encoded with: encoded2 do: [ :original :reencoded |
- 			self 
- 				assert: original charCode = reencoded charCode
- 				description: [ '{1} could not decode or re-encode {2} using #nextPut:toStream:. Instead, it yielded {3}.'
- 					format: {
- 						converterClass name.
- 						original charCode.
- 						reencoded charCode } ] ] ]!

Item was changed:
  ----- Method: MultiByteFileStreamTest>>testLineEndConversion (in category 'testing') -----
  testLineEndConversion
  
  	| failures | 
  	fileName := 'foolinendconversion.txt'.
  	failures := OrderedCollection new.
  	TextConverter allSubclassesDo: [ :textConverterClass |
  		textConverterClass encodingNames ifNotEmpty: [
  			#(cr lf crlf) do: [ :lineEndConvention |
  				self
  					testLineEndConvention: lineEndConvention
  					withConverter: textConverterClass
  					ifFail: [ :expectedResult :result |
  						failures add: {
  							textConverterClass.
  							lineEndConvention.
  							expectedResult.
  							result } ] ] ] ].
+ 	self assert: failures isEmpty!
- 	self assert: failures isEmpty. 
- 	"The code below is here to help you see why those cases fail"
- 	failures do: [ :failure |
- 		self
- 			testLineEndConvention: failure second
- 			withConverter: failure first
- 			ifFail: [ :expectedResult :result | self halt ] ]!

Item was changed:
+ UTFTextConverterWithByteOrderTest subclass: #UTF16TextConverterTest
- TestCase subclass: #UTF16TextConverterTest
  	instanceVariableNames: ''
  	classVariableNames: ''
  	poolDictionaries: ''
  	category: 'MultilingualTests-TextConversion'!

Item was added:
+ ----- Method: UTF16TextConverterTest>>characterWidthInBytes (in category 'constants') -----
+ characterWidthInBytes
+ 	"Only true for the test case. Beware of changing the unicode string"
+ 
+ 	^ 2!

Item was added:
+ ----- Method: UTF16TextConverterTest>>converterClass (in category 'constants') -----
+ converterClass
+ 
+ 	^ UTF16TextConverter!

Item was added:
+ UTFTextConverterWithByteOrderTest subclass: #UTF32TextConverterTest
+ 	instanceVariableNames: ''
+ 	classVariableNames: ''
+ 	poolDictionaries: ''
+ 	category: 'MultilingualTests-TextConversion'!

Item was added:
+ ----- Method: UTF32TextConverterTest>>characterWidthInBytes (in category 'constants') -----
+ characterWidthInBytes
+ 
+ 	^ 4!

Item was added:
+ ----- Method: UTF32TextConverterTest>>converterClass (in category 'constants') -----
+ converterClass
+ 
+ 	^ UTF32TextConverter!

Item was changed:
+ TestCase subclass: #UTF8TextConverterTest
- ClassTestCase subclass: #UTF8TextConverterTest
  	instanceVariableNames: ''
  	classVariableNames: ''
  	poolDictionaries: ''
  	category: 'MultilingualTests-TextConversion'!

Item was added:
+ ----- Method: UTF8TextConverterTest>>testBasicConversion (in category 'tests') -----
+ testBasicConversion
+ 	|converter originalText bytes decodedText |
+ 	originalText := self unicodeString.
+ 	converter := UTF8TextConverter new.
+ 	
+ 	"Default (ie useLittleEndian instvar nil)"
+ 	bytes := (originalText convertToWithConverter: converter).
+ 	decodedText := bytes convertFromWithConverter: converter.
+ 	
+ 	self assert: originalText equals: decodedText.
+ 	
+ 	!

Item was added:
+ ----- Method: UTF8TextConverterTest>>unicodeString (in category 'fixtures') -----
+ unicodeString
+ 
+ 	^ String newFrom: (#(19335 12403) collect: [:n | Character codePoint: n])!

Item was added:
+ TestCase subclass: #UTFTextConverterWithByteOrderTest
+ 	instanceVariableNames: ''
+ 	classVariableNames: ''
+ 	poolDictionaries: ''
+ 	category: 'MultilingualTests-TextConversion'!

Item was added:
+ ----- Method: UTFTextConverterWithByteOrderTest class>>isAbstract (in category 'as yet unclassified') -----
+ isAbstract
+ 
+ 	^ self = UTFTextConverterWithByteOrderTest!

Item was added:
+ ----- Method: UTFTextConverterWithByteOrderTest>>characterWidthInBytes (in category 'constants') -----
+ characterWidthInBytes
+ 
+ 	self subclassResponsibility!

Item was added:
+ ----- Method: UTFTextConverterWithByteOrderTest>>converterClass (in category 'constants') -----
+ converterClass
+ 
+ 	self subclassResponsibility!

Item was added:
+ ----- Method: UTFTextConverterWithByteOrderTest>>testByteOrders (in category 'tests') -----
+ testByteOrders
+ 	|converter originalText bytes decodedText |
+ 	originalText := 'test'.
+ 	converter := self converterClass new.
+ 	
+ 	"Default (ie useLittleEndian instvar nil)"
+ 	bytes := (originalText convertToWithConverter: converter).
+ 	decodedText := bytes convertFromWithConverter: converter.
+ 	
+ 	self assert: originalText equals: decodedText.
+ 	
+ 	"Little-endian"
+ 	converter useLittleEndian: true.
+ 
+ 	bytes := (originalText convertToWithConverter: converter).
+ 	decodedText := bytes convertFromWithConverter: converter.
+ 	
+ 	self assert: originalText equals: decodedText.
+ 	
+ 	"Big-endian"
+ 	converter useLittleEndian: false.
+ 
+ 	bytes := (originalText convertToWithConverter: converter).
+ 	decodedText := bytes convertFromWithConverter: converter.
+ 	
+ 	self assert: originalText equals: decodedText.
+ 	
+ 	!

Item was added:
+ ----- Method: UTFTextConverterWithByteOrderTest>>testByteOrdersMarks (in category 'tests') -----
+ testByteOrdersMarks
+ 	|converter originalText bytes decodedText |
+ 	originalText := self unicodeString.
+ 	converter := self converterClass new.
+ 	
+ 	converter useByteOrderMark: true.
+ 	bytes := (originalText convertToWithConverter: converter).
+ 	self assert: (self characterWidthInBytes * (self unicodeString size + 1)) equals: bytes size. 
+ 	decodedText := bytes convertFromWithConverter: converter.
+ 	
+ 	self assert: originalText equals: decodedText.
+ 	!

Item was added:
+ ----- Method: UTFTextConverterWithByteOrderTest>>testByteOrdersWithNonLatin (in category 'tests') -----
+ testByteOrdersWithNonLatin
+ 	|converter originalText bytes decodedText |
+ 	originalText := self unicodeString.
+ 	converter := self converterClass new.
+ 	
+ 	"Default (ie useLittleEndian instvar nil)"
+ 	bytes := (originalText convertToWithConverter: converter).
+ 	decodedText := bytes convertFromWithConverter: converter.
+ 	
+ 	self assert: originalText equals: decodedText.
+ 	
+ 	"Little-endian"
+ 	converter useLittleEndian: true.
+ 
+ 	bytes := (originalText convertToWithConverter: converter).
+ 	decodedText := bytes convertFromWithConverter: converter.
+ 	
+ 	self assert: originalText equals: decodedText.
+ 	
+ 	"Big-endian"
+ 	converter useLittleEndian: false.
+ 
+ 	bytes := (originalText convertToWithConverter: converter).
+ 	decodedText := bytes convertFromWithConverter: converter.
+ 	
+ 	self assert: originalText equals: decodedText.
+ 	
+ 	!

Item was added:
+ ----- Method: UTFTextConverterWithByteOrderTest>>unicodeString (in category 'fixtures') -----
+ unicodeString
+ 
+ 	^ String newFrom: (#(19335 12403) collect: [:n | Character codePoint: n])!