001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.namenode;
019
020import java.io.File;
021import java.io.IOException;
022import java.nio.file.FileVisitOption;
023import java.nio.file.FileVisitResult;
024import java.nio.file.Files;
025import java.nio.file.Path;
026import java.nio.file.SimpleFileVisitor;
027import java.nio.file.attribute.BasicFileAttributes;
028import java.util.Collections;
029
030import org.apache.commons.logging.Log;
031import org.apache.commons.logging.LogFactory;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hdfs.server.common.Storage;
034import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
035import org.apache.hadoop.hdfs.server.common.StorageInfo;
036
037import com.google.common.base.Preconditions;
038
039public abstract class NNUpgradeUtil {
040  
041  private static final Log LOG = LogFactory.getLog(NNUpgradeUtil.class);
042  
043  /**
044   * Return true if this storage dir can roll back to the previous storage
045   * state, false otherwise. The NN will refuse to run the rollback operation
046   * unless at least one JM or fsimage storage directory can roll back.
047   * 
048   * @param storage the storage info for the current state
049   * @param prevStorage the storage info for the previous (unupgraded) state
050   * @param targetLayoutVersion the layout version we intend to roll back to
051   * @return true if this JM can roll back, false otherwise.
052   * @throws IOException in the event of error
053   */
054  static boolean canRollBack(StorageDirectory sd, StorageInfo storage,
055      StorageInfo prevStorage, int targetLayoutVersion) throws IOException {
056    File prevDir = sd.getPreviousDir();
057    if (!prevDir.exists()) {  // use current directory then
058      LOG.info("Storage directory " + sd.getRoot()
059               + " does not contain previous fs state.");
060      // read and verify consistency with other directories
061      storage.readProperties(sd);
062      return false;
063    }
064
065    // read and verify consistency of the prev dir
066    prevStorage.readPreviousVersionProperties(sd);
067
068    if (prevStorage.getLayoutVersion() != targetLayoutVersion) {
069      throw new IOException(
070        "Cannot rollback to storage version " +
071        prevStorage.getLayoutVersion() +
072        " using this version of the NameNode, which uses storage version " +
073        targetLayoutVersion + ". " +
074        "Please use the previous version of HDFS to perform the rollback.");
075    }
076    
077    return true;
078  }
079
080  /**
081   * Finalize the upgrade. The previous dir, if any, will be renamed and
082   * removed. After this is completed, rollback is no longer allowed.
083   * 
084   * @param sd the storage directory to finalize
085   * @throws IOException in the event of error
086   */
087  static void doFinalize(StorageDirectory sd) throws IOException {
088    File prevDir = sd.getPreviousDir();
089    if (!prevDir.exists()) { // already discarded
090      LOG.info("Directory " + prevDir + " does not exist.");
091      LOG.info("Finalize upgrade for " + sd.getRoot()+ " is not required.");
092      return;
093    }
094    LOG.info("Finalizing upgrade of storage directory " + sd.getRoot());
095    Preconditions.checkState(sd.getCurrentDir().exists(),
096        "Current directory must exist.");
097    final File tmpDir = sd.getFinalizedTmp();
098    // rename previous to tmp and remove
099    NNStorage.rename(prevDir, tmpDir);
100    NNStorage.deleteDir(tmpDir);
101    LOG.info("Finalize upgrade for " + sd.getRoot()+ " is complete.");
102  }
103  
104  /**
105   * Perform any steps that must succeed across all storage dirs/JournalManagers
106   * involved in an upgrade before proceeding onto the actual upgrade stage. If
107   * a call to any JM's or local storage dir's doPreUpgrade method fails, then
108   * doUpgrade will not be called for any JM. The existing current dir is
109   * renamed to previous.tmp, and then a new, empty current dir is created.
110   *
111   * @param conf configuration for creating {@link EditLogFileOutputStream}
112   * @param sd the storage directory to perform the pre-upgrade procedure.
113   * @throws IOException in the event of error
114   */
115  static void doPreUpgrade(Configuration conf, StorageDirectory sd)
116      throws IOException {
117    LOG.info("Starting upgrade of storage directory " + sd.getRoot());
118
119    // rename current to tmp
120    renameCurToTmp(sd);
121
122    final Path curDir = sd.getCurrentDir().toPath();
123    final Path tmpDir = sd.getPreviousTmp().toPath();
124
125    Files.walkFileTree(tmpDir,
126      /* do not follow links */ Collections.<FileVisitOption>emptySet(),
127        1, new SimpleFileVisitor<Path>() {
128
129          @Override
130          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
131              throws IOException {
132
133            String name = file.getFileName().toString();
134
135            if (Files.isRegularFile(file)
136                && name.startsWith(NNStorage.NameNodeFile.EDITS.getName())) {
137
138              Path newFile = curDir.resolve(name);
139              Files.createLink(newFile, file);
140            }
141
142            return super.visitFile(file, attrs);
143          }
144        }
145    );
146  }
147
148  /**
149   * Rename the existing current dir to previous.tmp, and create a new empty
150   * current dir.
151   */
152  public static void renameCurToTmp(StorageDirectory sd) throws IOException {
153    File curDir = sd.getCurrentDir();
154    File prevDir = sd.getPreviousDir();
155    final File tmpDir = sd.getPreviousTmp();
156
157    Preconditions.checkState(curDir.exists(),
158        "Current directory must exist for preupgrade.");
159    Preconditions.checkState(!prevDir.exists(),
160        "Previous directory must not exist for preupgrade.");
161    Preconditions.checkState(!tmpDir.exists(),
162        "Previous.tmp directory must not exist for preupgrade."
163            + "Consider restarting for recovery.");
164
165    // rename current to tmp
166    NNStorage.rename(curDir, tmpDir);
167
168    if (!curDir.mkdir()) {
169      throw new IOException("Cannot create directory " + curDir);
170    }
171  }
172  
173  /**
174   * Perform the upgrade of the storage dir to the given storage info. The new
175   * storage info is written into the current directory, and the previous.tmp
176   * directory is renamed to previous.
177   * 
178   * @param sd the storage directory to upgrade
179   * @param storage info about the new upgraded versions.
180   * @throws IOException in the event of error
181   */
182  public static void doUpgrade(StorageDirectory sd, Storage storage)
183      throws IOException {
184    LOG.info("Performing upgrade of storage directory " + sd.getRoot());
185    try {
186      // Write the version file, since saveFsImage only makes the
187      // fsimage_<txid>, and the directory is otherwise empty.
188      storage.writeProperties(sd);
189
190      File prevDir = sd.getPreviousDir();
191      File tmpDir = sd.getPreviousTmp();
192      Preconditions.checkState(!prevDir.exists(),
193          "previous directory must not exist for upgrade.");
194      Preconditions.checkState(tmpDir.exists(),
195          "previous.tmp directory must exist for upgrade.");
196
197      // rename tmp to previous
198      NNStorage.rename(tmpDir, prevDir);
199    } catch (IOException ioe) {
200      LOG.error("Unable to rename temp to previous for " + sd.getRoot(), ioe);
201      throw ioe;
202    }
203  }
204
205  /**
206   * Perform rollback of the storage dir to the previous state. The existing
207   * current dir is removed, and the previous dir is renamed to current.
208   * 
209   * @param sd the storage directory to roll back.
210   * @throws IOException in the event of error
211   */
212  static void doRollBack(StorageDirectory sd)
213      throws IOException {
214    File prevDir = sd.getPreviousDir();
215    if (!prevDir.exists()) {
216      return;
217    }
218
219    File tmpDir = sd.getRemovedTmp();
220    Preconditions.checkState(!tmpDir.exists(),
221        "removed.tmp directory must not exist for rollback."
222            + "Consider restarting for recovery.");
223    // rename current to tmp
224    File curDir = sd.getCurrentDir();
225    Preconditions.checkState(curDir.exists(),
226        "Current directory must exist for rollback.");
227
228    NNStorage.rename(curDir, tmpDir);
229    // rename previous to current
230    NNStorage.rename(prevDir, curDir);
231
232    // delete tmp dir
233    NNStorage.deleteDir(tmpDir);
234    LOG.info("Rollback of " + sd.getRoot() + " is complete.");
235  }
236  
237}