1
0

simple-encoder.go 2.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112
  1. //+build ignore
  2. // Copyright 2015, Klaus Post, see LICENSE for details.
  3. //
  4. // Simple encoder example
  5. //
  6. // The encoder encodes a simgle file into a number of shards
  7. // To reverse the process see "simpledecoder.go"
  8. //
  9. // To build an executable use:
  10. //
  11. // go build simple-decoder.go
  12. //
  13. // Simple Encoder/Decoder Shortcomings:
  14. // * If the file size of the input isn't diviable by the number of data shards
  15. // the output will contain extra zeroes
  16. //
  17. // * If the shard numbers isn't the same for the decoder as in the
  18. // encoder, invalid output will be generated.
  19. //
  20. // * If values have changed in a shard, it cannot be reconstructed.
  21. //
  22. // * If two shards have been swapped, reconstruction will always fail.
  23. // You need to supply the shards in the same order as they were given to you.
  24. //
  25. // The solution for this is to save a metadata file containing:
  26. //
  27. // * File size.
  28. // * The number of data/parity shards.
  29. // * HASH of each shard.
  30. // * Order of the shards.
  31. //
  32. // If you save these properties, you should abe able to detect file corruption
  33. // in a shard and be able to reconstruct your data if you have the needed number of shards left.
  34. package main
  35. import (
  36. "flag"
  37. "fmt"
  38. "io/ioutil"
  39. "os"
  40. "path/filepath"
  41. "github.com/klauspost/reedsolomon"
  42. )
  43. var dataShards = flag.Int("data", 4, "Number of shards to split the data into, must be below 257.")
  44. var parShards = flag.Int("par", 2, "Number of parity shards")
  45. var outDir = flag.String("out", "", "Alternative output directory")
  46. func init() {
  47. flag.Usage = func() {
  48. fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
  49. fmt.Fprintf(os.Stderr, " simple-encoder [-flags] filename.ext\n\n")
  50. fmt.Fprintf(os.Stderr, "Valid flags:\n")
  51. flag.PrintDefaults()
  52. }
  53. }
  54. func main() {
  55. // Parse command line parameters.
  56. flag.Parse()
  57. args := flag.Args()
  58. if len(args) != 1 {
  59. fmt.Fprintf(os.Stderr, "Error: No input filename given\n")
  60. flag.Usage()
  61. os.Exit(1)
  62. }
  63. if *dataShards > 257 {
  64. fmt.Fprintf(os.Stderr, "Error: Too many data shards\n")
  65. os.Exit(1)
  66. }
  67. fname := args[0]
  68. // Create encoding matrix.
  69. enc, err := reedsolomon.New(*dataShards, *parShards)
  70. checkErr(err)
  71. fmt.Println("Opening", fname)
  72. b, err := ioutil.ReadFile(fname)
  73. checkErr(err)
  74. // Split the file into equally sized shards.
  75. shards, err := enc.Split(b)
  76. checkErr(err)
  77. fmt.Printf("File split into %d data+parity shards with %d bytes/shard.\n", len(shards), len(shards[0]))
  78. // Encode parity
  79. err = enc.Encode(shards)
  80. checkErr(err)
  81. // Write out the resulting files.
  82. dir, file := filepath.Split(fname)
  83. if *outDir != "" {
  84. dir = *outDir
  85. }
  86. for i, shard := range shards {
  87. outfn := fmt.Sprintf("%s.%d", file, i)
  88. fmt.Println("Writing to", outfn)
  89. err = ioutil.WriteFile(filepath.Join(dir, outfn), shard, os.ModePerm)
  90. checkErr(err)
  91. }
  92. }
  93. func checkErr(err error) {
  94. if err != nil {
  95. fmt.Fprintf(os.Stderr, "Error: %s", err.Error())
  96. os.Exit(2)
  97. }
  98. }