1
0

examples_test.go 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209
  1. package reedsolomon_test
  2. import (
  3. "bytes"
  4. "fmt"
  5. "io"
  6. "io/ioutil"
  7. "log"
  8. "math/rand"
  9. "github.com/klauspost/reedsolomon"
  10. )
  11. func fillRandom(p []byte) {
  12. for i := 0; i < len(p); i += 7 {
  13. val := rand.Int63()
  14. for j := 0; i+j < len(p) && j < 7; j++ {
  15. p[i+j] = byte(val)
  16. val >>= 8
  17. }
  18. }
  19. }
  20. // Simple example of how to use all functions of the Encoder.
  21. // Note that all error checks have been removed to keep it short.
  22. func ExampleEncoder() {
  23. // Create some sample data
  24. var data = make([]byte, 250000)
  25. fillRandom(data)
  26. // Create an encoder with 17 data and 3 parity slices.
  27. enc, _ := reedsolomon.New(17, 3)
  28. // Split the data into shards
  29. shards, _ := enc.Split(data)
  30. // Encode the parity set
  31. _ = enc.Encode(shards)
  32. // Verify the parity set
  33. ok, _ := enc.Verify(shards)
  34. if ok {
  35. fmt.Println("ok")
  36. }
  37. // Delete two shards
  38. shards[10], shards[11] = nil, nil
  39. // Reconstruct the shards
  40. _ = enc.Reconstruct(shards)
  41. // Verify the data set
  42. ok, _ = enc.Verify(shards)
  43. if ok {
  44. fmt.Println("ok")
  45. }
  46. // Output: ok
  47. // ok
  48. }
  49. // This demonstrates that shards can be arbitrary sliced and
  50. // merged and still remain valid.
  51. func ExampleEncoder_slicing() {
  52. // Create some sample data
  53. var data = make([]byte, 250000)
  54. fillRandom(data)
  55. // Create 5 data slices of 50000 elements each
  56. enc, _ := reedsolomon.New(5, 3)
  57. shards, _ := enc.Split(data)
  58. err := enc.Encode(shards)
  59. if err != nil {
  60. panic(err)
  61. }
  62. // Check that it verifies
  63. ok, err := enc.Verify(shards)
  64. if ok && err == nil {
  65. fmt.Println("encode ok")
  66. }
  67. // Split the data set of 50000 elements into two of 25000
  68. splitA := make([][]byte, 8)
  69. splitB := make([][]byte, 8)
  70. // Merge into a 100000 element set
  71. merged := make([][]byte, 8)
  72. // Split/merge the shards
  73. for i := range shards {
  74. splitA[i] = shards[i][:25000]
  75. splitB[i] = shards[i][25000:]
  76. // Concencate it to itself
  77. merged[i] = append(make([]byte, 0, len(shards[i])*2), shards[i]...)
  78. merged[i] = append(merged[i], shards[i]...)
  79. }
  80. // Each part should still verify as ok.
  81. ok, err = enc.Verify(shards)
  82. if ok && err == nil {
  83. fmt.Println("splitA ok")
  84. }
  85. ok, err = enc.Verify(splitB)
  86. if ok && err == nil {
  87. fmt.Println("splitB ok")
  88. }
  89. ok, err = enc.Verify(merged)
  90. if ok && err == nil {
  91. fmt.Println("merge ok")
  92. }
  93. // Output: encode ok
  94. // splitA ok
  95. // splitB ok
  96. // merge ok
  97. }
  98. // This demonstrates that shards can xor'ed and
  99. // still remain a valid set.
  100. //
  101. // The xor value must be the same for element 'n' in each shard,
  102. // except if you xor with a similar sized encoded shard set.
  103. func ExampleEncoder_xor() {
  104. // Create some sample data
  105. var data = make([]byte, 25000)
  106. fillRandom(data)
  107. // Create 5 data slices of 5000 elements each
  108. enc, _ := reedsolomon.New(5, 3)
  109. shards, _ := enc.Split(data)
  110. err := enc.Encode(shards)
  111. if err != nil {
  112. panic(err)
  113. }
  114. // Check that it verifies
  115. ok, err := enc.Verify(shards)
  116. if !ok || err != nil {
  117. fmt.Println("falied initial verify", err)
  118. }
  119. // Create an xor'ed set
  120. xored := make([][]byte, 8)
  121. // We xor by the index, so you can see that the xor can change,
  122. // It should however be constant vertically through your slices.
  123. for i := range shards {
  124. xored[i] = make([]byte, len(shards[i]))
  125. for j := range xored[i] {
  126. xored[i][j] = shards[i][j] ^ byte(j&0xff)
  127. }
  128. }
  129. // Each part should still verify as ok.
  130. ok, err = enc.Verify(xored)
  131. if ok && err == nil {
  132. fmt.Println("verified ok after xor")
  133. }
  134. // Output: verified ok after xor
  135. }
  136. // This will show a simple stream encoder where we encode from
  137. // a []io.Reader which contain a reader for each shard.
  138. //
  139. // Input and output can be exchanged with files, network streams
  140. // or what may suit your needs.
  141. func ExampleStreamEncoder() {
  142. dataShards := 5
  143. parityShards := 2
  144. // Create a StreamEncoder with the number of data and
  145. // parity shards.
  146. rs, err := reedsolomon.NewStream(dataShards, parityShards)
  147. if err != nil {
  148. log.Fatal(err)
  149. }
  150. shardSize := 50000
  151. // Create input data shards.
  152. input := make([][]byte, dataShards)
  153. for s := range input {
  154. input[s] = make([]byte, shardSize)
  155. fillRandom(input[s])
  156. }
  157. // Convert our buffers to io.Readers
  158. readers := make([]io.Reader, dataShards)
  159. for i := range readers {
  160. readers[i] = io.Reader(bytes.NewBuffer(input[i]))
  161. }
  162. // Create our output io.Writers
  163. out := make([]io.Writer, parityShards)
  164. for i := range out {
  165. out[i] = ioutil.Discard
  166. }
  167. // Encode from input to output.
  168. err = rs.Encode(readers, out)
  169. if err != nil {
  170. log.Fatal(err)
  171. }
  172. fmt.Println("ok")
  173. // OUTPUT: ok
  174. }